Last active
August 27, 2022 15:31
-
-
Save Haruite/6d081c356a72feb5f0aba77b9178c6f7 to your computer and use it in GitHub Desktop.
Revisions
-
Haruite revised this gist
Aug 19, 2022 . 1 changed file with 13 additions and 16 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -711,20 +711,6 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: if 'tid' not in data: data['tid'] = -1 torrents.append(data) if f1 == 1 and self.m_conf['enable']: @@ -1316,6 +1302,7 @@ def limit_speed(self): try: self.update_tid() self.update_upload() continue except: pass else: @@ -1498,6 +1485,17 @@ def fix_next_announce(self): self.to['next_announce'] = next_announce self.to['next_announce_is_true'] = True if 'last_announce_time' in self.to and 'date' in self.to: self.to['next_announce'] = int(self.to['last_announce_time'] + self.announce_interval - time()) + 1 while self.to['next_announce'] < 0: self.to['next_announce'] += self.announce_interval if self.to['tid'] != -1 and 'date' in self.to and 'uploaded_before' not in self.to: if abs(time() + self.to['next_announce'] - self.announce_interval - self.to['time_added']) < 180: self.to['uploaded_before'] = self.to['uploaded'] else: self.to['uploaded_before'] = '0 B' def optimize_announce_time(self): """尽量把完成前最后一次汇报时间调整到最合适的点，粗略计算，没有严格讨论问题。 @@ -1741,5 +1739,4 @@ def info_from_peer_list(self): else: logger.error('The program will do nothing') os._exit(0) -
Aug 18, 2022 . 2 changed files with 57 additions and 41 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -17,8 +17,8 @@ uc_max = -1 # 单个魔法 uc 使用量最大值，如果为 -1 则不检查，否则超过不放魔法 uc_24_max = -1 # 24h 内 uc 使用最大值，如果为 -1 则不检查，否则超过不放魔法 tid_max = -1 # 种子 id 最大值，如果为 -1 则不检查，否则超过不放魔法 user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' \ 'Chrome/103.0.5060.134 Safari/537.36 Edg/103.0.1264.77' # 用户代理 proxies = { # 'http': 'http://127.0.0.1:10809', 'https': 'http://127.0.0.1:10809' } # 代理 log_path = f'{os.path.splitext(__file__)[0]}.log' # 日志路径 @@ -42,14 +42,14 @@ def __init__(self): def get_pro(tr): pro = {'ur': 1.0, 'dr': 1.0} pro_dict = {'free': {'dr': 0.0}, 'twoup': {'ur': 2.0}, 'halfdown': {'dr': 0.5}, 'thirtypercent': {'dr': 0.3}} if tr.get('class'): # 高亮显示 [pro.update(data) for key, data in pro_dict.items() if key in tr['class'][0]] td = tr.tr and tr.select('tr')[1].td or tr.select('td')[1] pro_dict_1 = {'free': {'dr': 0.0}, '2up': {'ur': 2.0}, '50pct': {'dr': 0.5}, '30pct': {'dr': 0.3}, 'custom': {}} for img in td.select('img') or []: # 图标显示 if not [pro.update(data) for key, data in pro_dict_1.items() if key in img['class'][0]]: pro[{'arrowup': 'ur', 'arrowdown': 'dr'}[img['class'][0]]] = float(img.next_element.text[:-1].replace(',', '.')) for span in td.select('span') or []: # 标记显示 [pro.update(data) for key, data in pro_dict.items() if key in (span.get('class') and span['class'][0] or '')] return list(pro.values()) @@ -73,10 +73,9 @@ def run(self): _dr = 1 if dr >= data['pro'][1] else dr if not (_ur == 1 and _dr == 1): _data = {'hours': hours, 'torrent': tid, 'user': user, 'ur': _ur, 'dr': _dr} if any(tid == _info['torrent'] and _ur <= _info['ur'] and _dr >= _info['dr'] for _info in self.magic_info): continue if data['seeder_num'] > 0: try: self.magic(_data) @@ -131,3 +130,4 @@ def magic(self, _data): logger.exception(e) finally: sleep(interval)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -30,7 +30,7 @@ 这其实是软件的问题，不过我估计这种现象还挺普遍的，所以大致解决了这个麻烦 已知问题： 暂时没有大的问题 """ @@ -504,7 +504,7 @@ def run(self): self.torrents_info = self.get_info_from_client() if self.m_conf['enable'] or self.l_conf['enable']: self.fix_next_announce() if self.m_conf['enable']: # 顺序不能颠倒 self.magic() if self.l_conf['enable']: self.limit_speed() @@ -603,7 +603,7 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: table1 = soup1.find('table', {'width': '90%'}) torrent['date'] = table1.time.attrs.get('title') or table1.time.text for tr1 in table1: if tr1.td.text in ['种子信息', '種子訊息', 'Torrent Info', 'Информация о торренте', 'Torrent Info', 'Информация о торренте']: torrent['_id'] = tr1.tr.contents[-2].contents[1].strip() @@ -618,6 +618,7 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: _info: Dict[str, Dict[str, Any]] = {} # 存放客户端获取的当前种子信息 _ids: set = set({}) # 存放所有需要知道是否在客户端的种子 hash [_ids.add(torrent['_id']) for torrent in torrents if '_id' in torrent and 'in_client' not in torrent] all_connected = True if len(_ids) > 0 and len(t_client) > 1: @@ -632,19 +633,20 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: _info.update(future.result()) except Exception as e: logger.exception(e) all_connected = False else: for _id in list(_ids): for hash_id, data in _info.items(): if hash_id == _id: _ids.remove(_id) [to.update({'in_client': True}) for to in torrents if to.get('_id') == _id] if len(_ids) == 0: executor._threads.clear() break if all_connected: # 如果有些客户端连接不上，可能有些种子不能确定是否客户端 [to.update({'in_client': False}) for to in torrents if '_id' in to and 'in_client' not in to] return torrents def get_info_from_client(self) -> List[Dict[str, Any]]: @@ -710,19 +712,19 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: data['tid'] = -1 # ********** 最后，更新一些额外的信息 if 'last_announce_time' in data and 'date' in data: # 先修复 next_announce，不然没法更新 uploaded_before self.to = data data['next_announce'] = int(data['last_announce_time'] + self.announce_interval - time()) + 1 while data['next_announce'] < 0: data['next_announce'] += self.announce_interval if data['tid'] != -1 and 'uploaded_before' not in data and 'date' in data: self.to = data if abs(time() + data['next_announce'] - self.announce_interval - data['time_added']) < 180: data['uploaded_before'] = data['uploaded'] else: data['uploaded_before'] = '0 B' torrents.append(data) if f1 == 1 and self.m_conf['enable']: @@ -803,7 +805,7 @@ def magic(self): if self.to['tid'] == -1: continue if self.client is None and '_id' in self.to: if self.to.get('in_client'): continue if self.is_new: if self.m_conf['magic_new']: @@ -1037,8 +1039,13 @@ def check_time(self, data: Dict[str, Any]) -> Union[bool, None]: self.print(f"{_begin}is about to re-announce, passed") return True if 'total_size' not in self.to: if 'in_client' not in self.to and self.is_new: '''新种，本该交给客户端的线程放魔法，但客户端失联，希望等待恢复 但如果一直失联超过一定时间，就必须放魔法了，时间是估算的，如果是 10G 带宽还要把时间估小一点''' if self.deta > self.byte(self.to['size'], 0) / 55 / 1024 ** 2: return True return '''除此之外，就是已确定不在客户端端下载的种子''' if self.to['seeder_num'] > 0: self.print(f'{_begin}Seeder-num > 0, passed') return True @@ -1290,9 +1297,10 @@ def limit_speed(self): 解释一下什么是超速。tracker 并不知道种子的上传速度情况，因为种子每次汇报的只有上传量、下载量和剩余完成量， 而 peer 列表的瞬时速度，是由最近两次汇报的上传量差/最近两次汇报的时间差计算的， 只要这个值小于 50M/s，就会把两次汇报上传量的差值加到账号的实际上传，乘以种子优惠比率加到虚拟上传，否则的话就不计算。 也就是说只要在相邻两次汇报之间的上传量不超过 (50M/s * 两次汇报的时间间隔) 就行。 通常情况下种子会以固定的周期向 tracker 汇报，不超速情况下两次汇报间的最大上传量是固定的， 只有在快要传满的时候限速就行。但问题是，种子完成时也会向 tracker 汇报， 这个时间是未知的，如果种子下载时间小于 30 分钟而限速是按照 30 分钟汇报间隔计算，那么在完成汇报时就会超速。 解决这个问题有两种方法，一种在完成前最后一次汇报后进行特殊处理，检测到平均速度即将超过 50M/s 就限速， 这样一来不管什么时候完成都不会超速；另一种就是在快要完成时限速下载以延后完成时间， @@ -1387,13 +1395,20 @@ def limit_download_speed(self): else: '''平均速度已降到 50M/s 以下，解除限速，之似乎发现 tracker 计算的时间精度比秒更精确？ 无论如何 next_announce 是个整数必须 +1s''' self.client.set_upload_limit(self.to['_id'], 51200) self.client.set_download_limit(self.to['_id'], -1) self.to['max_download_speed'] = -1 logger.info(f'Removed download speed limit of torrent {self.to["tid"]}.') for _ in range(30): sleep(1) try: if self.client.torrent_status(self.to['_id'], ['state'])['state'] == 'Seeding': self.client.set_upload_limit(self.to['_id'], -1) self.to['max_upload_speed'] = -1 return except: pass logger.error(f"Torrent {self.to['tid']} | failed to remove upload limit") def limit_upload_speed(self): if 10 < self.to['eta'] + 10 < self.to['next_announce']: @@ -1491,7 +1506,7 @@ def optimize_announce_time(self): 假设这个种子不限速时上传速度是一个稳定的数值，那么最后一次汇报时间有一个点能使完成时间延长最多。 但实际并非总是如人意，比如最后一次定期汇报时间刚好在完成时，就没有任何可以延长下载时间的余地。 这个函数就是解决这个问题，在合适的时间强制汇报来调整完成前最后一次汇报时间。""" i = int(300 / self.client.connect_interval) + 1 if 'detail_progress' not in self.to: self.to['detail_progress'] = deque(maxlen=i) @@ -1508,7 +1523,7 @@ def optimize_announce_time(self): earliest 是计算的最早能强制汇报且不超速的时间。 如果最佳汇报时间可以强制汇报并且不超速，直接汇报就行，实际并非总是如此。 有可能最早能汇报的时间在最佳时间点之后，这时候就需要比较在最早能汇报的时间汇报和不强制汇报''' complete_time = (self.to['total_size'] - self.to['total_done']) / dlspeed + time() perfect_time = complete_time - self.announce_interval * 52428800 / upspeed if self.this_up / self.this_time > 52428800: @@ -1727,3 +1742,4 @@ def info_from_peer_list(self): else: logger.error('The program will do nothing') os._exit(0)
-
Aug 6, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1466,7 +1466,7 @@ def limit_upload_speed(self): def fix_next_announce(self): """目前已知 lt1.2.16/1.2.17/2.0.6/2.0.7 next_announce 可能与实际不和， 通过查询 peerlist 计算上传汇报时间并得到实际值，可能存在一定误差""" for self.to in filter(lambda to: 'tid' in to and 'date' in to, self.torrents_info): if time() - self.to['time_added'] < self.announce_interval: if time() - self.to['time_added'] + self.to['next_announce'] - self.announce_interval < -600: if 'last_announce_time' not in self.to and not self.to.get('next_announce_is_true'): -
Aug 6, 2022 . 1 changed file with 6 additions and 6 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -502,12 +502,12 @@ def run(self): while True: try: self.torrents_info = self.get_info_from_client() if self.m_conf['enable'] or self.l_conf['enable']: self.fix_next_announce() if self.m_conf['enable']: self.magic() if self.l_conf['enable']: self.limit_speed() except Exception as e: logger.exception(e) finally: -
Aug 6, 2022 . 1 changed file with 25 additions and 49 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -502,10 +502,12 @@ def run(self): while True: try: self.torrents_info = self.get_info_from_client() if self.m_conf['enable'] or self.l_conf['enable']: self.fix_next_announce() if self.m_conf['enable']: self.magic() if self.l_conf['enable']: self.limit_speed() except Exception as e: logger.exception(e) finally: @@ -1285,10 +1287,6 @@ def change_mode(cls) -> int: def limit_speed(self): """将两次汇报间的平均速度限制到 50M/s 以下 解释一下什么是超速。tracker 并不知道种子的上传速度情况，因为种子每次汇报的只有上传量、下载量和剩余完成量， 而 peer 列表的瞬时速度，是由最近两次汇报的上传量差/最近两次汇报的时间差计算的， 只要这个值小于 50M/s，就会把两次汇报上传量的差值加到账号的实际上传，乘以种子优惠比率加到虚拟上传，否则的话就不计算。 @@ -1298,20 +1296,7 @@ def limit_speed(self): 这个时间是未知的，如果种子下载时间小于 30 分钟而限速是按照 30 分钟汇报间隔计算，那么在完成汇报时就会超速。 解决这个问题有两种方法，一种在完成前最后一次汇报后进行特殊处理，检测到平均速度即将超过 50M/s 就限速， 这样一来不管什么时候完成都不会超速；另一种就是在快要完成时限速下载以延后完成时间， 但无论如何到下一次定期汇报时间点也是要汇报的。这里使用的是第二种方法。""" f1 = 0 for self.to in self.torrents_info: @@ -1336,8 +1321,6 @@ def limit_speed(self): logger.error(f"Could not find 'last_get_time' of torrent {self.to['tid']}") continue if time() - self.this_time + 2 > self.to['last_get_time'] and f1 == 0: # 刚汇报完，更新上次汇报的上传量 if self.to['total_uploaded'] > 0: @@ -1483,21 +1466,22 @@ def limit_upload_speed(self): def fix_next_announce(self): """目前已知 lt1.2.16/1.2.17/2.0.6/2.0.7 next_announce 可能与实际不和， 通过查询 peerlist 计算上传汇报时间并得到实际值，可能存在一定误差""" for self.to in filter(lambda to: 'tid' in to, self.torrents_info): if time() - self.to['time_added'] < self.announce_interval: if time() - self.to['time_added'] + self.to['next_announce'] - self.announce_interval < -600: if 'last_announce_time' not in self.to and not self.to.get('next_announce_is_true'): next_announce = self.to['next_announce'] if next_announce > 3: logger.debug(f"Unexpected next announce time of torrent {self.to['tid']}") self.to['last_announce_time'] = time() self.info_from_peer_list() if abs(self.to['last_announce_time'] + 900 - time() - next_announce) < 3: logger.debug('Caused by manually re-announce') del self.to['last_announce_time'] if 'true_downloaded' in self.to: del self.to['true_downloaded'] self.to['next_announce'] = next_announce self.to['next_announce_is_true'] = True def optimize_announce_time(self): """尽量把完成前最后一次汇报时间调整到最合适的点，粗略计算，没有严格讨论问题。 @@ -1506,13 +1490,8 @@ def optimize_announce_time(self): 那么可以把这个种子到完成时的平均速度按 50M/s 计算，要获得尽可能多的上传量则需要使完成时间尽可能延后。 假设这个种子不限速时上传速度是一个稳定的数值，那么最后一次汇报时间有一个点能使完成时间延长最多。 但实际并非总是如人意，比如最后一次定期汇报时间刚好在完成时，就没有任何可以延长下载时间的余地。 这个函数就是解决这个问题，在合适的时间强制汇报来调整最后一次汇报时间。""" i = int(300 / self.client.connect_interval) + 1 if 'detail_progress' not in self.to: self.to['detail_progress'] = deque(maxlen=i) @@ -1529,10 +1508,7 @@ def optimize_announce_time(self): earliest 是计算的最早能强制汇报且不超速的时间。 如果最佳汇报时间可以强制汇报并且不超速，直接汇报就行，实际并非总是如此。 有可能最早能汇报的时间在最佳时间点之后，这时候就需要比较在最早能汇报的时间汇报和不强制汇报。''' complete_time = (self.to['total_size'] - self.to['total_done']) / dlspeed + time() perfect_time = complete_time - self.announce_interval * 52428800 / upspeed if self.this_up / self.this_time > 52428800: -
Aug 6, 2022 . 1 changed file with 224 additions and 133 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -18,7 +18,7 @@ 以及删掉多余的 min_download_reduced 和 max_uc_peer_gb_reduced 当然，也可以设置 min_tid 和 min_leecher_num 很大让所有种子都被判断为旧种 不提供配置检查，因为不是重点（其实是因为懒） 所有功能和绝大部分语句都已得到测试 已修复问题: * 使用 hdd 因为上传速度太快而失联，导致限速无法运行，加入了一种比较暴力的方法，在失联的时候网卡限速 @@ -502,10 +502,10 @@ def run(self): while True: try: self.torrents_info = self.get_info_from_client() if self.l_conf['enable']: self.limit_speed() if self.m_conf['enable']: self.magic() except Exception as e: logger.exception(e) finally: @@ -847,6 +847,7 @@ def magic_new(self): hours = int((1 - progress) / progress * seed_time / 3600) + 1 hours = min(max(hours, 24), 360) # ********** 检查每个规则，符合就生成魔法 # ********** 把上传的魔法和下载的魔法拆开 # ********** 时长、范围相同的情况下，上传和下载的魔法可以分开放也可以合并，uc 使用量是一样的 # ********** 具体是否合并取决于时间检查 @@ -868,6 +869,7 @@ def magic_new(self): self.print(f"torrent {self.to['tid']} | rule {rule} - Failed. " f"Reason: {data}") # ********** 合并由规则生成的一系列魔法 # ********** 其实是支持给另一个人放魔法的，但问题是网页显示的是自己的优惠，如果先给自己放了魔法的话可能就不会给另一个人放了 # ********** 解决的办法是直接查种子的优惠历史，而且只能查一次，反正我是不打算写这个... # ********** 至于多个魔法嘛，没有这样的设计，不仅耗费 uc，而且会使程序变得很复杂和让人迷惑 @@ -891,7 +893,7 @@ def magic_new(self): elif data['hours'] > down_data['hours']: down_data = data # 合并上传和下载的魔法，如果时长范围一致，比如说 2.33x↑1x↓ 和 1x↑0x↓ 合并成 2.33x↑0x↓，以及检查是否重复施加魔法 if up_data != {} and self.check_time(up_data): magic_data = up_data if down_data != {} and self.check_time(down_data): @@ -1026,7 +1028,8 @@ def expected_cost(self, rule: Dict[str, Any]) -> float: # 估计 uc 消耗量 e_cost = m * c * pow(s, 0.5) * (pow(2 * ur - 2, 1.5) + pow(2 - 2 * dr, 2)) * pow(ttl, -0.8) * pow(h, 0.5) return e_cost def check_time(self, data: Dict[str, Any]) -> Union[bool, None]: """优化放魔法时间，如果到了放魔法的时间则返回 True""" _begin = f"torrent {self.to['tid']} | magic {data}: " if self.to.get('about_to_reannounce'): self.print(f"{_begin}is about to re-announce, passed") @@ -1232,6 +1235,15 @@ def total_uc_cost(cls) -> Tuple[int, int]: # 计算 24h 和 72h uc 使用量之 @classmethod def change_mode(cls) -> int: """根据 uc 使用量选取规则。 为什么要动态规则呢，可能是因为我有选择困难症，不知道怎么放魔法好。 其实可以优化 uc 使用，使用量少就多放些魔法，否则就少放些魔法。 新种大部分有地图炮魔法的时候，魔法系数稳步增长，也就是说同样情况下魔法越来越贵。 这是因为全站虚拟分享率在增长(看看公式里的 divergence 系数)， 没有 free 的时候魔法系数就会下跌，也就是说放魔法还起到调节魔法价格的作用， 这也是为什么我不希望总是全部放 free 的原因""" m_conf = conf['magic'] old_mode = cls.mode uc_24, uc_72 = cls.total_uc_cost() @@ -1271,45 +1283,63 @@ def change_mode(cls) -> int: return cls.mode def limit_speed(self): """将两次汇报间的平均速度限制到 50M/s 以下 为什么一个魔法脚本里面会有限速呢，首先是因为两者都需要定期读取客户端的信息以及从网页获取的信息， 有很多重复代码，另外由于限速涉及到强制汇报，强制汇报前最好先放魔法吧？ 虽然大部分人都不用限速就是了(真的这个只是给刷子用的) 解释一下什么是超速。tracker 并不知道种子的上传速度情况，因为种子每次汇报的只有上传量、下载量和剩余完成量， 而 peer 列表的瞬时速度，是由最近两次汇报的上传量差/最近两次汇报的时间差计算的， 只要这个值小于 50M/s，就会把两次汇报上传量的差值加到账号的实际上传，乘以种子优惠比率加到虚拟上传，否则的话就不计算。 也就是说只要在一个两次汇报之间的上传量不超过 (50M/s * 两次汇报的时间间隔) 就行。 通常情况下种子会以固定的周期向 tracker 汇报，问题是，种子完成时也会向 tracker 汇报， 这个时间是未知的，如果种子下载时间小于 30 分钟而限速是按照 30 分钟汇报间隔计算，那么在完成汇报时就会超速。 解决这个问题有两种方法，一种在完成前最后一次汇报后进行特殊处理，检测到平均速度即将超过 50M/s 就限速， 这样一来不管什么时候完成都不会超速；另一种就是在快要完成时限速下载以延后完成时间， 但无论如何到下一次定期汇报时间点也是要汇报的。这里使用的是第二种方法。需要知道三个值。 1.当前的上传量，直接从 deluge 获取 total_uploaded 2.上次汇报的上传量，deluge 似乎没有提供这个数值。不过可以从网页上获取，一般精度是够用的。 通常是从个人信息的下载种子页面获取，但是有一个问题，那里的数据是分段统计可能存在漏记， 必要时还会从 peer 列表获取上传量(不要搞抓包，原因你懂的) 3.上次汇报到现在的时间，通过汇报间隔 - next_announce 得到。 (通常 u2 给出的新种的最小汇报间隔是 30 分钟，超过一个月的种子是 60 分钟 libtorrent 会遵守这个最小汇报时间，但如果设置的最小汇报间隔还要大的话就按照大的来("最小")。 汇报间隔多长都行，但如果汇报间隔过长会被 tracker 当做重新连接，上传量不会计算。) 这里使用的一些方法在不懂的人看来很像是作弊，但实际上并没有，所以解释一下。 顺便提一下似乎也有简单方法可以提高分享率，有兴趣可以研究一下。 我就不放出来了，因为对其他用户不公平""" f1 = 0 for self.to in self.torrents_info: if self.to['tid'] == -1: '''旧种子默认不限速，因为没有查详情页不知道 id，不知道上传汇报的上传量。 但是当上传速度超过 50M/s 后就有超速可能，这时候就需要查找 id''' if self.to['upload_payload_rate'] > 52428800 and not self.to.get('404'): logger.debug(f"Try to find tid of {self.to['_id']} --- ") try: self.update_tid() self.update_upload() except: pass else: self.print(f"Will not limit speed of {self.to['_id']}") continue if 'date' not in self.to: # 按理说是不会有这种情况的 logger.error(f"Could not find 'date' of torrent {self.to['tid']}") continue if 'last_get_time' not in self.to: # 按理说是不会有这种情况的 logger.error(f"Could not find 'last_get_time' of torrent {self.to['tid']}") continue self.fix_next_announce() if time() - self.this_time + 2 > self.to['last_get_time'] and f1 == 0: # 刚汇报完，更新上次汇报的上传量 if self.to['total_uploaded'] > 0: try: self.update_upload() @@ -1320,129 +1350,189 @@ def limit_speed(self): if self.l_conf['variable_announce_interval']: self.optimize_announce_time() self.limit_download_speed() if self.this_time < 0: # 汇报后 tracker 还没有返回 continue self.limit_upload_speed() def limit_download_speed(self): if self.to['max_download_speed'] == -1: if self.this_time > 2 and self.this_up / self.this_time > 52428800: ps = 0 m_t = self.min_time if self.to['max_upload_speed'] != -1: '''上传限速时，如果限速值很低，给其他 peer 上传速度低， 其他 peer 给自己的上传速度也会很低，所以会严重拖慢下载进度，eta 值会变大。 但是出种后其他 peer 变成做种状态，这时候的上传策略一般是根据下载者的下载速度， 跟下载者的上传速度没有关系，由于先前没有下载限速，所以这时候种子可能突然变成满速下载， 不仅下载时间短而且客户端可能变得很难连接，可能导致限速失败。 所以这里在上传限速时检查其他 peer 的进度，在其他 peer 完成前提前下载限速。''' m_t = 2 * self.min_time p0 = 1 - 1610612736 / self.to['total_size'] try: for peer in self.client.torrent_status(self.to['_id'], ['peers'])['peers']: if peer['progress'] > p0: ps += 1 except: pass if 0 < self.to['eta'] <= m_t or self.to['max_upload_speed'] != -1 and ps > 20: # 平均速度超过 50M/s 并且快要完成，开始下载限速 max_download_speed = (self.to['total_size'] - self.to['total_done']) / ( self.this_up / 52428800 - self.this_time + 30) / 1024 self.client.set_download_limit(self.to['_id'], max_download_speed) logger.warning(f'Begin to limit download speed of torrent {self.to["tid"]}.' f' Value ------- {max_download_speed:.2f}K') elif self.this_time > 0: if self.this_up / self.this_time >= 52428800: # 已有下载限速，调整限速值 if self.to['download_payload_rate'] / 1024 < 2 * self.to['max_download_speed']: max_download_speed = (self.to['total_size'] - self.to['total_done']) / ( self.this_up / 52428800 - self.this_time + 60) / 1024 max_download_speed = min(max_download_speed, 512000) if max_download_speed > 1.5 * self.to['max_download_speed']: max_download_speed = 1.5 * self.to['max_download_speed'] self.client.set_download_limit(self.to['_id'], max_download_speed) logger.debug(f'Change the max download speed of torrent {self.to["tid"]} ' f'to {max_download_speed:.2f}K') elif max_download_speed < self.to['max_download_speed']: max_download_speed = max_download_speed / 1.5 self.client.set_download_limit(self.to['_id'], max_download_speed) logger.debug(f'Change the max download speed of torrent {self.to["tid"]} ' f'to {max_download_speed:.2f}K') else: '''平均速度已降到 50M/s 以下，解除限速，之似乎发现 tracker 计算的时间精度比秒更精确？ 无论如何 next_announce 是个整数必须 +1s''' self.client.set_download_limit(self.to['_id'], -1) self.to['max_download_speed'] = -1 logger.info(f'Removed download speed limit of torrent {self.to["tid"]}.') if self.to['max_upload_speed'] != -1: self.client.set_upload_limit(self.to['_id'], -1) self.to['max_upload_speed'] = -1 logger.info(f'Removed upload speed limit of torrent {self.to["tid"]}.') def limit_upload_speed(self): if 10 < self.to['eta'] + 10 < self.to['next_announce']: eta = self.to['eta'] + 10 else: eta = self.to['next_announce'] '''eta 代表到下次汇报之前还可以正常上传的时间， 如果完成时间在下次周期汇报之前，那么完成时就会汇报，到下次汇报的时间就是到完成的时间， 虽然可能通过下载限速延长完成时间，但是在延长的那段时间由于已经出种并且下载速度有限制， 通常并不能上传很多，所以可以正常上传的时间就按照完成时间计算''' if self.to['max_upload_speed'] == -1: res = self.min_time / self.m_conf['min_connect_times_before_announce'] * 2 \ * self.to['upload_payload_rate'] if self.this_up + res + 6291456 * eta > self.announce_interval * 52428800: '''上次汇报到现在的上传量即将超过一个汇报周期内允许的不超速的最大值，开始上传限速. 限速值不要太低，太低会跟不上进度影响之后的上传''' self.client.set_upload_limit(self.to['_id'], 6144) logger.warning(f'Begin to limit upload speed of torrent {self.to["tid"]}. Value ------- {6144}K') self.to['_t'] = time() else: # 已经开始上传限速，调整限速值 if self.to['max_upload_speed'] == 5120: # 在 optimize_announce_time 用到了这个，也可以手动限速到 5120k 等待汇报 if self.this_up / self.this_time < 52428800 and self.this_time >= 900: self.re_an() self.client.set_upload_limit(self.to['_id'], -1) logger.info('Average upload speed below 50MiB/s, remove 5120K up-limit') elif self.this_time < 120: # 已经汇报完，解除上传限速 self.client.set_upload_limit(self.to['_id'], -1) logger.info(f'Removed upload speed limit of torrent {self.to["tid"]}.') elif self.to['upload_payload_rate'] / 1024 < 2 * self.to['max_upload_speed']: max_upload_speed = (self.announce_interval * 52428800 - self.this_up) / (eta + 10) / 1024 '''计算上传限速值。把 +10 变成 +1，甚至可以限速到 49.999，不过也很容易超（不知道下载用固态会不会好点）''' if max_upload_speed > 51200: self.client.set_upload_limit(self.to['_id'], -1) logger.info(f'Removed upload speed limit of torrent {self.to["tid"]}.') elif max_upload_speed < 0: # 上传量超过了一个汇报间隔内不超速的最大值 if self.this_up / self.this_time < 209715200: if self.this_time >= 900: self.client.reannounce(self.to['_id']) logger.error(f'Failed to limit upload speed limit of torrent {self.to["tid"]} ' f'because the upload exceeded') else: self.client.set_upload_limit(self.to['_id'], 1) elif 8192 < max_upload_speed < 51200 and eta > 180: # 调整限速值减小余量，deluge 上传量一般比限速值低 self.client.set_upload_limit(self.to['_id'], 51200) logger.info(f'Set 51200K upload limit for torrent {self.to["tid"]}') elif 8192 < max_upload_speed < 16384 and eta > 60: self.client.set_upload_limit(self.to['_id'], 16384) logger.info(f'Set 16384K upload limit for torrent {self.to["tid"]}') else: if self.announce_interval * 52428800 - self.this_up > 94371840 and max_upload_speed < 3072: max_upload_speed = 3072 # 这个速度下载还不会卡住 if self.announce_interval * 52428800 - self.this_up > 31457280 and max_upload_speed < 1024: max_upload_speed = 1024 # 这个速度在出种前会卡死下载 if self.to['max_upload_speed'] != max_upload_speed: if max_upload_speed == 5120: max_upload_speed = 5119 self.client.set_upload_limit(self.to['_id'], max_upload_speed) if max_upload_speed in [3072, 1024]: logger.debug(f'Set {max_upload_speed}K upload limit to torrent {self.to["tid"]}') elif '_t' not in self.to or '_t' in self.to and time() - self.to['_t'] > 120: # 2 分钟输出一次，当然也可以直接输出(改成 > 0)，不过我觉得有点频繁 logger.debug(f'Change the max upload speed for torrent {self.to["tid"]} ' f'to {max_upload_speed:.2f}K') self.to['_t'] = time() def fix_next_announce(self): """目前已知 lt1.2.16/1.2.17/2.0.6/2.0.7 next_announce 可能与实际不和， 通过查询 peerlist 计算上传汇报时间并得到实际值，可能存在一定误差""" if time() - self.to['time_added'] < self.announce_interval: if time() - self.to['time_added'] + self.to['next_announce'] - self.announce_interval < -600: if 'last_announce_time' not in self.to and not self.to.get('next_announce_is_true'): next_announce = self.to['next_announce'] if next_announce > 3: logger.debug(f"Unexpected next announce time of torrent {self.to['tid']}") self.to['last_announce_time'] = time() self.info_from_peer_list() if abs(self.to['last_announce_time'] + 900 - time() - next_announce) < 3: logger.debug('Caused by manually re-announce') del self.to['last_announce_time'] if 'true_downloaded' in self.to: del self.to['true_downloaded'] self.to['next_announce'] = next_announce self.to['next_announce_is_true'] = True def optimize_announce_time(self): """尽量把完成前最后一次汇报时间调整到最合适的点，粗略计算，没有严格讨论问题。 解释一下，假设一个种子的下载时间超过汇报时长，并且这个种子每次汇报前都经过限速并且两次汇报间的平均速度接近 50M/s， 那么可以把这个种子到完成时的平均速度按 50M/s 计算，要获得尽可能多的上传量则需要使完成时间尽可能延后。 假设这个种子不限速时上传速度是一个稳定的数值，那么最后一次汇报时间有一个点能使完成时间延长最多。 举例说明。如果汇报间隔是一个小时，上传速度是 250M/s， 那么 12 分钟内就可以传完一个汇报间隔允许的最大的不超速的上传量， 最后一次汇报的最佳时间就是完成前 12 分钟，在这个时间点汇报，这个种子可以获得 48 分钟 50M/s 的超额上传量。 但实际并非总是如人意，比如最后一次定期汇报时间刚好在完成时，就没有任何可以延长下载时间的余地。 这个函数就是解决这个问题，在合适的时间强制汇报来调整最后一次汇报时间。 强制汇报前必须限速，不能按照默认的汇报间隔计算。为了简化问题，将种子上传速度为 5120k, 平均速度降到 50M/s 后就强制汇报。""" i = int(300 / self.client.connect_interval) + 1 if 'detail_progress' not in self.to: self.to['detail_progress'] = deque(maxlen=i) self.to['detail_progress'].append((self.to['total_uploaded'], self.to['total_done'], time())) if len(self.to['detail_progress']) != i or self.this_time < 30 or self.to['max_upload_speed'] == 5120: return _list = self.to['detail_progress'] '''计算 5 分钟内平均下载速度和平均上传速度''' upspeed = (_list[i - 1][0] - _list[0][0]) / (_list[i - 1][2] - _list[0][2]) dlspeed = (_list[i - 1][1] - _list[0][1]) / (_list[i - 1][2] - _list[0][2]) if upspeed > 52428800 and dlspeed > 0 and _list[0][1] != 0: '''complete_time 是估计的完成时间， perfect_time 是估计的最佳的最后一次汇报时间， earliest 是计算的最早能强制汇报且不超速的时间。 如果最佳汇报时间可以强制汇报并且不超速，直接汇报就行，实际并非总是如此。 有可能最早能汇报的时间在最佳时间点之后，这时候就需要比较在最早能汇报的时间汇报和不强制汇报， 因为看上去有效总上传量跟最后一次汇报时间是线性关系，只需要取其端点。但由于还存在一个问题， 也就是两次汇报间的间隔不能小于 15 分钟，所以严格来说要考虑完成前最后两次汇报， 这里简化处理，只考虑之后有一次汇报的情况''' complete_time = (self.to['total_size'] - self.to['total_done']) / dlspeed + time() perfect_time = complete_time - self.announce_interval * 52428800 / upspeed if self.this_up / self.this_time > 52428800: @@ -1451,7 +1541,7 @@ def optimize_announce_time(self): earliest = time() if earliest - (time() - self.this_time) < 900: return if earliest > perfect_time: if time() >= earliest: if (self.this_up + upspeed * 20) / self.this_time > 52428800: self.re_an() @@ -1462,7 +1552,7 @@ def optimize_announce_time(self): self.to['max_upload_speed'] = 5120 logger.info(f"Set 5120K upload limit for torrent {self.to['tid']}, waiting for re-announce") else: if time() - self.this_time > perfect_time: return _eta1 = complete_time - earliest if _eta1 < 120: @@ -1490,6 +1580,7 @@ def re_an(self): self.to['about_to_reannounce'] = False def update_tid(self): """根据 hash 搜索种子 id""" url = f'https://u2.dmhy.org/torrents.php?incldead=0&spstate=0' \ f'&inclbookmarked=0&search={self.to["_id"]}&search_area=5&search_mode=0' try: -
Jun 27, 2022 . 1 changed file with 6 additions and 9 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -630,6 +630,7 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: _info.update(future.result()) except Exception as e: logger.exception(e) return torrents for _id in list(_ids): for hash_id, data in _info.items(): @@ -722,13 +723,8 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: torrents.append(data) if f1 == 1 and self.m_conf['enable']: t_client[0].magic() self.last_connect = time() return torrents @@ -804,8 +800,9 @@ def magic(self): for self.to in self.torrents_info: if self.to['tid'] == -1: continue if self.client is None and '_id' in self.to: if 'in_client' not in self.to or self.to['in_client']: continue if self.is_new: if self.m_conf['magic_new']: self.magic_new() -
Jun 23, 2022 . 1 changed file with 5 additions and 5 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -548,10 +548,10 @@ def rq(self, method: str, url: str, timeout: Union[int, float] = 10, retries: in elif i == retries - 1: raise Exception(f'Failed to request... ' f'method: {method}, url: {url}, kw: {kw}' f' ------ status code: {code}') elif code in [502, 503]: delay = int(html.headers.get('Retry-After') or '30') logger.error(f'Will attempt to request {url} after {delay}s') sleep(delay) except Exception as e: if i == retries - 1: @@ -747,7 +747,7 @@ def get_pro(tr: Tag) -> List[Union[int, float]]: pro_dict_1 = {'free': {'dr': 0.0}, '2up': {'ur': 2.0}, '50pct': {'dr': 0.5}, '30pct': {'dr': 0.3}, 'custom': {}} for img in td.select('img') or []: # 图标显示 if not [pro.update(data) for key, data in pro_dict_1.items() if key in img['class'][0]]: pro[{'arrowup': 'ur', 'arrowdown': 'dr'}[img['class'][0]]] = float(img.next.text[:-1].replace(',', '.')) for span in td.select('span') or []: # 标记显示 [pro.update(data) for key, data in pro_dict.items() if key in (span.get('class') and span['class'][0] or '')] @@ -798,7 +798,7 @@ def deta(self) -> int: # 返回种子发布时间与当前的时间差 def get_tz(soup: Tag) -> str: tz_info = soup.find('a', {'href': 'usercp.php?action=tracker#timezone'})['title'] pre_suf = [['时区', '，点击修改。'], ['時區', '，點擊修改。'], ['Current timezone is ', ', click to change.']] return [tz_info[len(pre):-len(suf)].strip() for pre, suf in pre_suf if tz_info.startswith(pre)][0] def magic(self): for self.to in self.torrents_info: @@ -1571,7 +1571,7 @@ def info_from_peer_list(self): self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(int, tr.contents[10].string.split(':'))) self.to['last_announce_time'] = time() - idle self.to['next_announce'] = self.announce_interval - idle + 1 if self.to['next_announce'] < 0: -
Jun 23, 2022 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -691,7 +691,7 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: if 'tid' not in data: if f1 == 0: try: t_client[0].get_info_from_web() '''没有用 locate_client，是为了避免多线程同时使用同一个 deluge 对象''' for to in t_client[0].torrents_info: if to.get('_id') == data['_id']: @@ -1119,7 +1119,7 @@ def check_duplicate(self, data: Dict[str, Any]) -> Union[bool, None]: if time() - _info['ts'] < _info['hours'] * 3600: if data['ur'] <= _info['ur'] and data['dr'] >= _info['dr']: return True if 'last_get_time' in self.to and time() - self.to['last_get_time'] < 0.01 or not self.is_new: return try: page = self.rq('get', f'https://u2.dmhy.org/details.php?id={self.to["tid"]}&hit=1').text -
Jun 23, 2022 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -622,8 +622,8 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: # 由于可能出现不可预料的延迟，采用线程任务 with ThreadPoolExecutor(max_workers=len(t_client) - 1) as executor: """修复 Segmentation fault 发现 deepcopy 不能解决问题，单独给第一个线程创建对象算了，如果我的想法是对的那么问题已经解决了 如果还是异常退出，可以用 monitor.py，检测到脚本退出后自动运行脚本""" futures = [executor.submit(cl.downloading_torrents_info, self.status_keys) for cl in self.clients] for future in as_completed(futures): try: -
Jun 23, 2022 . 1 changed file with 40 additions and 48 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -589,15 +589,10 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: torrent.update(_torrent) break if tid > self.m_conf['min_tid'] or torrent['leecher_num'] > self.m_conf['min_leecher_num']: # 旧种子不需要知道 hash，因为不需要在客户端的线程放魔法 # ********** 第三步，已有信息查不到 hash，获取种子详细页 # 这一步是将种子 tid 与 _id 联系起来的入口 if '_id' not in torrent: detail_page = self.rq('get', f'https://u2.dmhy.org/details.php?id={tid}&hit=1').text @@ -620,44 +615,32 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Detect whether a new torrent is in BT client""" _info: Dict[str, Dict[str, Any]] = {} # 存放客户端获取的当前种子信息 _ids: set = set({}) # 存放所有需要知道是否在客户端的种子 hash [_ids.add(torrent['_id']) for torrent in torrents if '_id' in torrent and 'in_client' not in torrent] if len(_ids) > 0 and len(t_client) > 1: # 由于可能出现不可预料的延迟，采用线程任务 with ThreadPoolExecutor(max_workers=len(t_client) - 1) as executor: """修复 Segmentation fault 发现 deepcopy 不能解决问题，单独给第一个线程创建对象算了，如果我的想法是对的那么问题已经解决了 如果还是异常退出，可以用 monitor.py，检测到脚本退出后自动运行脚本""" futures = [executor.submit(cl.downloading_torrents_info, self.status_keys) for cl in self.clients] for future in as_completed(futures): try: _info.update(future.result()) except Exception as e: logger.exception(e) for _id in list(_ids): for hash_id, data in _info.items(): if hash_id == _id: _ids.remove(_id) [to.update({'in_client': True}) for to in torrents if to.get('_id') == _id] if len(_ids) == 0: executor._threads.clear() break [to.update({'in_client': False}) for to in torrents if '_id' in to and 'in_client' not in to] return torrents @@ -689,23 +672,27 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: torrent['first_seed_time'] = time() torrent.update(data) data.update(torrent) # 等价于 [data.setdefault(key, val) for key, val in torrent.items()] break # ********** 第三步，更新网页获取的种子信息，这一步也是必做，因为要更新上传下载量 for _torrent in t_client[0].torrents_info: if _id == _torrent.get('_id') or data.get('tid') == _torrent['tid']: if '_id' not in _torrent: _torrent['_id'] = _id _torrent['in_client'] = True data.update(_torrent) '''但是这会导致另一个潜在的 bug，如果单独限速，t_client[0] 是不工作的 更新 uploaded 时需要更新 t_client[0] 的 torrents_info 的对应信息, 否则到了这里 uploaded 会变为原来的值''' break # ********** 第四步，已有信息都查不到，获取下载页面分析 if 'tid' not in data: if f1 == 0: try: t_client[0].torrents_info = t_client[0].get_info_from_web() '''没有用 locate_client，是为了避免多线程同时使用同一个 deluge 对象''' for to in t_client[0].torrents_info: if to.get('_id') == data['_id']: to['in_client'] = True @@ -726,18 +713,20 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: data['uploaded_before'] = data['uploaded'] else: data['uploaded_before'] = '0 B' if 'last_announce_time' in data and 'date' in data: self.to = data data['next_announce'] = int(data['last_announce_time'] + self.announce_interval - time()) + 1 while data['next_announce'] < 0: data['next_announce'] += self.announce_interval torrents.append(data) if f1 == 1: t_client[0].torrents_info = [to for to in t_client[0].torrents_info if not ('_id' in to and 'in_client' not in to)] '''因为没有 locate_client，t_client[0].torrents_info 可能包含其他客户端的种子信息， 如果没有 in_client 信息，放魔法时会被当做旧种子，这些种子信息需要去除''' if self.m_conf['enable']: t_client[0].magic() @@ -1543,8 +1532,6 @@ def update_upload(self): tmp_info.append(self.to) if self.to['total_uploaded'] - self.byte(data['uploaded'], 1) > \ 300 * 1024 ** 2 * (self.this_time + 2): self.to['true_uploaded'] = data['uploaded'] tmp_info.append(self.to) if data['uploaded'].split(' ')[0] != '0': @@ -1576,10 +1563,12 @@ def info_from_peer_list(self): if 'true_uploaded' in self.to: self.to['true_uploaded'] = tr.contents[1].string self.to['true_downloaded'] = tr.contents[4].string if self.to['true_uploaded'] == self.to['uploaded']: del self.to['true_uploaded'] del self.to['true_downloaded'] else: self.print(f"Some upload of torrent {self.to['tid']} was not calculated by tracker") self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(lambda n: int(n), tr.contents[10].string.split(':'))) @@ -1653,6 +1642,9 @@ def info_from_peer_list(self): try: with ThreadPoolExecutor(max_workers=len(t_client)) as _executor: _futures = {_executor.submit(_c.run): _c.client for _c in t_client} """因为 deluge 很容易失联，如果有多个客户端，要分配多个线程让各个客户端时间上不受牵制。 第一个线程客户端是 None，这个线程的任务就是定期爬网页以及放魔法(对不在客户端的种子)， 单独开限速时，这个线程什么也不做。之后的线程每个都对应有一个客户端，给在客户端的种子放魔法以及限速""" for _future in as_completed(_futures): try: _future.result() -
Jun 22, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1584,7 +1584,7 @@ def info_from_peer_list(self): if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(lambda n: int(n), tr.contents[10].string.split(':'))) self.to['last_announce_time'] = time() - idle self.to['next_announce'] = self.announce_interval - idle + 1 if self.to['next_announce'] < 0: self.to['next_announce'] = 0 -
Jun 22, 2022 . 1 changed file with 1 addition and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1579,6 +1579,7 @@ def info_from_peer_list(self): self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if self.to['true_uploaded'] == self.to['uploaded']: del self.to['true_uploaded'] del self.to['true_downloaded'] if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(lambda n: int(n), tr.contents[10].string.split(':'))) -
Jun 22, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1575,7 +1575,7 @@ def info_from_peer_list(self): if 'true_uploaded' in self.to: self.to['true_uploaded'] = tr.contents[1].string self.to['true_downloaded'] = tr.contents[4].string self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if self.to['true_uploaded'] == self.to['uploaded']: del self.to['true_uploaded'] -
Jun 22, 2022 . 1 changed file with 4 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1541,7 +1541,8 @@ def update_upload(self): if time() - self.this_time + 10 > self.to['last_get_time']: if 'true_uploaded' in self.to or 'last_announce_time' in self.to: tmp_info.append(self.to) if self.to['total_uploaded'] - self.byte(data['uploaded'], 1) > \ 300 * 1024 ** 2 * (self.this_time + 2): self.print(f"Some upload of torrent {self.to['tid']} was not calculated by tracker, " f"try to get upload from peer list") self.to['true_uploaded'] = data['uploaded'] @@ -1576,6 +1577,8 @@ def info_from_peer_list(self): self.to['true_uploaded'] = tr.contents[1].string self.to['true_downloaded'] = tr.contents[1].string self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if self.to['true_uploaded'] == self.to['uploaded']: del self.to['true_uploaded'] if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(lambda n: int(n), tr.contents[10].string.split(':'))) -
Jun 22, 2022 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1538,10 +1538,10 @@ def update_upload(self): data = {'uploaded': tr.contents[6].get_text(' '), 'last_get_time': time()} if 'date' in self.to and 'last_get_time' in self.to: if time() - self.this_time + 10 > self.to['last_get_time']: if 'true_uploaded' in self.to or 'last_announce_time' in self.to: tmp_info.append(self.to) if self.to['total_uploaded'] - self.byte(data['uploaded'], 1) > 300 * 1024 ** 2 * (self.this_time + 2): self.print(f"Some upload of torrent {self.to['tid']} was not calculated by tracker, " f"try to get upload from peer list") self.to['true_uploaded'] = data['uploaded'] -
Jun 22, 2022 . 1 changed file with 115 additions and 18 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -20,11 +20,17 @@ 不提供配置检查，因为不是重点（其实是因为懒） 除了多个客户端，所有功能和绝大部分语句都已得到测试 已修复问题: * 使用 hdd 因为上传速度太快而失联，导致限速无法运行，加入了一种比较暴力的方法，在失联的时候网卡限速 * 多线程同时使用同一个 deluge 对象连接时可能引起 segmentation fault，给第一个线程使用不同的 deluge 对象，就不存在共用了 * 各种语言、时间显示类型为发生或者过去、优惠显示类型加高亮或者图标或者标记或者没有，以及任意时区 * 如果因为某些原因导致下载种子页面的上传量没被统计，会使用 peer 列表的上传量(正常情况下超速也是会统计的， 已知有三种情况会使部分流量不被统计，第一种是被当作同时下载，第二种是清空 tracker，还有一种是汇报超时，似乎还有其它不明情况) * 有可能部分 libtorrent 版本有问题，next_announce 显示时间与实际不符，会通过查询 peer 列表的空闲时间来计算。 这其实是软件的问题，不过我估计这种现象还挺普遍的，所以大致解决了这个麻烦 已知问题： 无 """ @@ -35,6 +41,7 @@ import paramiko import pytz from functools import reduce from copy import deepcopy from datetime import datetime from collections import deque @@ -308,6 +315,10 @@ conf = yaml.load(config, yaml.FullLoader) class ConfigError(Exception): pass class BtClient(metaclass=ABCMeta): # 这个基类规定了 BT 客户端必须实现的功能 @abstractmethod @@ -715,6 +726,12 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: data['uploaded_before'] = data['uploaded'] else: data['uploaded_before'] = '0 B' if 'last_announce_time' in data and 'date' in data: self.to = data data['next_announce'] = int(data['last_announce_time'] + self.announce_interval - time()) + 1 while data['next_announce'] < 0: data['next_announce'] += self.announce_interval torrents.append(data) @@ -991,10 +1008,10 @@ def expected_add(self, rule: Dict[str, Any]) -> Union[int, float]: # 期望的 urr = rule['ur'] - self.to['promotion'][0] if 'total_uploaded' in self.to: e_up = self.to['total_uploaded'] / (self.to['total_done'] + 1024) * self.to['total_size'] e_add = (e_up - self.byte(self.to.get('true_uploaded') or self.to['uploaded'], 0)) * urr else: uploaded = self.byte(self.to['uploaded'], 0) downloaded = self.byte(self.to.get('true_downloaded') or self.to['downloaded'], 0) size = self.byte(self.to['size'], 0) if downloaded < 1024 ** 2: e_add = self.m_conf['default_ratio'] * size * urr @@ -1007,7 +1024,7 @@ def expected_reduce(self, rule: Dict[str, Any]) -> Union[int, float]: # 期望 size = self.to['total_size'] else: size = self.byte(self.to['size'], 0) return (size - self.byte(self.to.get('true_downloaded') or self.to['downloaded'], 0)) * (1 - rule['dr']) def expected_cost(self, rule: Dict[str, Any]) -> float: # 估计 uc 消耗量 c = self.coefficient @@ -1160,7 +1177,7 @@ def this_up(self) -> int: # 当前种子自上次汇报的上传量 _before = self.byte(self.to['uploaded_before'], 1) else: _before = 0 _now = self.byte(self.to.get('true_uploaded') or self.to['uploaded'], -1) return self.to['total_uploaded'] - _now + _before @property @@ -1289,6 +1306,23 @@ def limit_speed(self): if 'last_get_time' not in self.to: logger.error(f"Could not find 'last_get_time' of torrent {self.to['tid']}") continue if time() - self.to['time_added'] < self.announce_interval: if time() - self.to['time_added'] + self.to['next_announce'] - self.announce_interval < -600: if 'last_announce_time' not in self.to and not self.to.get('next_announce_is_true'): next_announce = self.to['next_announce'] if next_announce > 3: logger.debug(f"Unexpected next announce time of torrent {self.to['tid']}") self.to['last_announce_time'] = time() self.info_from_peer_list() if abs(self.to['last_announce_time'] + 900 - time() - next_announce) < 3: logger.debug('Caused by manually re-announce') del self.to['last_announce_time'] if 'true_downloaded' in self.to: del self.to['true_downloaded'] self.to['next_announce'] = next_announce self.to['next_announce_is_true'] = True if time() - self.this_time + 10 > self.to['last_get_time'] and f1 == 0: # 更新上次汇报的上传量 if self.to['total_uploaded'] > 0: try: @@ -1465,6 +1499,8 @@ def re_an(self): self.to = _to sleep(1) self.client.reannounce(self.to['_id']) if 'last_announce_time' in self.to: self.to['last_announce_time'] = time() self.to['about_to_reannounce'] = False def update_tid(self): @@ -1486,25 +1522,78 @@ def update_tid(self): logger.error(e) def update_upload(self): tmp_to = self.to try: page = self.rq('get', f'https://u2.dmhy.org/getusertorrentlistajax.php?userid={conf["uid"]}&type=leeching').text table = BeautifulSoup(page.replace('\n', ''), 'lxml').table if not table: return tmp_info = [] for tr in table.contents[1:]: tid = int(tr.contents[1].a['href'][15:-6]) for self.to in self.torrents_info: if self.to['tid'] != tid: continue data = {'uploaded': tr.contents[6].get_text(' '), 'last_get_time': time()} if 'date' in self.to and 'last_get_time' in self.to: if time() - self.this_time + 2 > self.to['last_get_time']: if 'true_uploaded' in self.to or 'last_announce_time' in self.to: tmp_info.append(self.to) if self.to['total_uploaded'] - self.byte(data['uploaded'], 1) > 300 * 1024 ** 2 * self.this_time: self.print(f"Some upload of torrent {self.to['tid']} was not calculated by tracker, " f"try to get upload from peer list") self.to['true_uploaded'] = data['uploaded'] tmp_info.append(self.to) if data['uploaded'].split(' ')[0] != '0': self.print(f"Last announce upload of torrent {tid} is {data['uploaded']}") self.to.update(data) [_torrent.update(data) for _torrent in t_client[0].torrents_info if _torrent['tid'] == tid] for self.to in tmp_info: self.info_from_peer_list() except Exception as e: logger.exception(e) finally: self.to = tmp_to def info_from_peer_list(self): """Fix incorrect upload and next announce""" try: peer_list = self.rq('get', f"https://u2.dmhy.org/viewpeerlist.php?id={self.to['tid']}").text tables = BeautifulSoup(peer_list.replace('\n', ' '), 'lxml').find_all('table') except Exception as e: logger.error(e) return for table in tables or []: for tr in filter(lambda _tr: 'nowrap' in str(_tr), table): if tr.get('bgcolor'): if 'true_uploaded' in self.to: self.to['true_uploaded'] = tr.contents[1].string self.to['true_downloaded'] = tr.contents[1].string self.print(f"Actual upload of torrent {self.to['tid']} is {self.to['true_uploaded']}") if 'last_announce_time' in self.to: idle = reduce(lambda a, b: a * 60 + b, map(lambda n: int(n), tr.contents[10].string.split(':'))) self.to['last_announce_time'] = time() - idle self.to['next_announce'] = int(self.announce_interval - idle) - 1 if self.to['next_announce'] < 0: self.to['next_announce'] = 0 if __name__ == '__main__': modes = conf['magic']['modes'] if modes and len(modes) > 1: for i in range(len(modes) - 1): if modes[i]['uc_limit']['24_max'] < modes[i + 1]['uc_limit']['24_min']: raise ConfigError() if modes[i]['uc_limit']['72_max'] < modes[i + 1]['uc_limit']['72_min']: raise ConfigError() log_path = conf.get('log_path') or f'{os.path.splitext(__file__)[0]}.log' data_path = conf.get('data_path') or f'{os.path.splitext(__file__)[0]}.data.txt' logger.remove(handler_id=0) # 默认有一个 sys.stderr handler 会输出 debug 信息，需要清除 @@ -1535,6 +1624,14 @@ def update_upload(self): _client = MagicAndLimit(local_client) else: _client = MagicAndLimit(Deluge(**_c)) try: min_announce_interval = _client.client.ltconfig.get_settings()['min_announce_interval'] except: min_announce_interval = 300 if _c['min_announce_interval'] != min_announce_interval: raise ConfigError() t_client[0].clients.append(Deluge(**_c)) t_client.append(_client) -
Jun 18, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1495,7 +1495,7 @@ def update_upload(self): tid = int(tr.contents[1].a['href'][15:-6]) for torrent in self.torrents_info: if torrent['tid'] == tid: data = {'uploaded': tr.contents[6].get_text(' '), 'last_get_time': time()} torrent.update(data) if torrent['uploaded'] != '0 B' and tid == self.to['tid']: self.print(f"Last announce upload of torrent {tid} is {torrent['uploaded']}") -
Jun 18, 2022 . 1 changed file with 3 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1471,11 +1471,13 @@ def update_tid(self): url = f'https://u2.dmhy.org/torrents.php?incldead=0&spstate=0' \ f'&inclbookmarked=0&search={self.to["_id"]}&search_area=5&search_mode=0' try: soup = BeautifulSoup(self.rq('get', url).text.replace('\n', ''), 'lxml') table = soup.select('table.torrents') if table: self.to['tid'] = int(table[0].contents[1].contents[1].a['href'][15:-6]) date = table[0].contents[1].contents[3].time self.to['date'] = date.get('title') or date.get_text(' ') self.to['tz'] = self.get_tz(soup) logger.debug(f"{self.to['_id']} --> {self.to['tid']}") else: self.to['404'] = True -
Jun 18, 2022 . 1 changed file with 2 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1474,7 +1474,8 @@ def update_tid(self): table = BeautifulSoup(self.rq('get', url).text.replace('\n', ''), 'lxml').select('table.torrents') if table: self.to['tid'] = int(table[0].contents[1].contents[1].a['href'][15:-6]) date = table[0].contents[1].contents[3].time self.to['date'] = date.get('title') or date.get_text(' ') logger.debug(f"{self.to['_id']} --> {self.to['tid']}") else: self.to['404'] = True -
Jun 18, 2022 . 1 changed file with 22 additions and 21 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,29 +1,30 @@ """ 给下载中的种子放魔法，python3.6 及以上应该能运行 依赖：pip3 install PyYAML requests bs4 lxml deluge-client loguru func-timeout 支持客户端 deluge，其它的客户端自己去写类吧 支持配置多个客户端，可以任意停止和重新运行 检查重复，检查 uc 使用量，尽可能减少爬网页的次数 放魔法区分新种和旧种，因为新种魔法使用量太多，支持自定义魔法规则 不支持对单个种子同时施加一个以上的上传或者下载魔法 可以根据 24h 和 72h 的 uc 使用量自动切换规则 根据客户端的种子信息调整放魔法的时间，最小化 uc 使用量 对下载的种子进行限速，防止上传失效 用法： 按 yaml 语法修改 config，填上必填信息，按注释说明修改其它信息 至少应该填上 uid 和 cookie，默认为只给旧种放魔法 如果要给所有下载的种子放 free，将 magic_new 改为 True，default_mode 改为 4 以及删掉多余的 min_download_reduced 和 max_uc_peer_gb_reduced 当然，也可以设置 min_tid 和 min_leecher_num 很大让所有种子都被判断为旧种 不提供配置检查，因为不是重点（其实是因为懒） 除了多个客户端，所有功能和绝大部分语句都已得到测试 已知的问题: * 如果因为某些原因导致下载页面的上传量没被统计，使用限速会出现误差，也就是限少了(正常情况下超速也是会统计的， 已知有三种情况会使部分流量不被统计，第一种是被当作同时下载，第二种是清空 tracker，还有一种是汇报超时，似乎还有其它不明情况) 实际上 peer 列表的上传量和客户端的上传量是完全符合的，但并没有这样处理的打算 * 有可能部分 libtorrent 版本有问题，next_announce 显示时间与实际不符，如果有问题建议使用 lt 1.2.15 及以下版本 """ -
Jun 16, 2022 . 2 changed files with 37 additions and 24 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -15,6 +15,7 @@ ] class Monitor: def __init__(self, interpreter, script): self.interpreter = interpreter @@ -23,31 +24,43 @@ def __init__(self, interpreter, script): def run(self): while True: try: i = 0 for pid in psutil.pids(): try: process = psutil.Process(pid) if self.interpreter in process.name(): if any(self.script in s for s in process.cmdline()): i = 1 except psutil.NoSuchProcess: pass except Exception as e: logger.exception(e) if i == 0: logger.error(f'Process {self.script} is down') if self.child: self.child.terminate() self.child = pexpect.spawn(f'{self.interpreter} {self.script}', logfile=sys.stdout.buffer) logger.warning(f'Run command | {self.interpreter} {self.script}') try: self.child.expect(pexpect.EOF, timeout=9223372036) except Exception as e: logger.exception(e) elif self.child: try: self.child.expect(pexpect.EOF, timeout=9223372036) except Exception as e: logger.exception(e) except Exception as e: logger.exception(e) finally: sleep(10) if __name__ == '__main__': log_path = f'{os.path.splitext(__file__)[0]}.log' logger.add(level='DEBUG', sink=log_path, encoding='utf-8', rotation="5 MB") monitors = [Monitor(interpreter, script) for interpreter, script in COMMANDS] try: with ThreadPoolExecutor(max_workers=len(monitors)) as executor: [executor.submit(monitor.run) for monitor in monitors] This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -561,11 +561,11 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: torrent['tid'] = tid = int(conts[1].a['href'][15:-6]) torrent['category'] = int(conts[0].a['href'][26:]) torrent['title'] = conts[1].a.b.text torrent['size'] = conts[2].get_text(' ') torrent['seeder_num'] = int(conts[3].string) torrent['leecher_num'] = int(conts[4].string) torrent['uploaded'] = conts[6].get_text(' ') torrent['downloaded'] = conts[7].get_text(' ') torrent['promotion'] = self.get_pro(tr) # ************ 第二步，和 torrent_info 已有信息合并 @@ -1473,7 +1473,7 @@ def update_tid(self): table = BeautifulSoup(self.rq('get', url).text.replace('\n', ''), 'lxml').select('table.torrents') if table: self.to['tid'] = int(table[0].contents[1].contents[1].a['href'][15:-6]) self.to['date'] = table[0].contents[1].contents[3].time.get_text(' ') logger.debug(f"{self.to['_id']} --> {self.to['tid']}") else: self.to['404'] = True @@ -1491,7 +1491,7 @@ def update_upload(self): tid = int(tr.contents[1].a['href'][15:-6]) for torrent in self.torrents_info: if torrent['tid'] == tid: data = {'upload': tr.contents[6].get_text(' '), 'last_get_time': time()} torrent.update(data) if torrent['uploaded'] != '0 B' and tid == self.to['tid']: self.print(f"Last announce upload of torrent {tid} is {torrent['uploaded']}") -
Jun 16, 2022 . 1 changed file with 6 additions and 10 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -609,13 +609,10 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: _info: Dict[str, Dict[str, Any]] = {} # 存放客户端获取的当前种子信息 _ids: set = set({}) # 存放所有需要知道是否在客户端的种子 hash for torrent in torrents: # ********** 第一步，查询客户端线程已有信息是否有此 hash if '_id' in torrent and 'in_client' not in torrent: if not [torrent.update({'in_client': True}) for c in t_client[1:] for _torrent in c.torrents_info if torrent['_id'] == _torrent['_id']]: _ids.add(torrent['_id']) if len(_ids) > 0: @@ -1115,9 +1112,8 @@ def check_duplicate(self, data: Dict[str, Any]) -> Union[bool, None]: if time() - _info['ts'] < _info['hours'] * 3600: if data['ur'] <= _info['ur'] and data['dr'] >= _info['dr']: return True if 'last_get_time' in self.to and time() - self.to['last_get_time'] < 0.01: return try: page = self.rq('get', f'https://u2.dmhy.org/details.php?id={self.to["tid"]}&hit=1').text soup = BeautifulSoup(page.replace('\n', ''), 'lxml') -
Jun 16, 2022 . 1 changed file with 11 additions and 31 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -579,13 +579,8 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: # ********** 第三步，查询客户端的线程的信息 if '_id' not in torrent: [torrent.update({'_id': _torrent['_id'], 'in_client': True}) for c in t_client[1:] for _torrent in c.torrents_info if tid == _torrent.get('tid')] if tid > self.m_conf['min_tid'] or torrent['leecher_num'] > self.m_conf['min_leecher_num']: # 旧种子不需要知道 hash，因为不需要在客户端的线程放魔法 @@ -618,12 +613,8 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: # ********** 第一步，查询客户端线程已有信息是否有此 hash if 'in_client' not in torrent: [torrent.update({'in_client': True}) for c in t_client[1:] for _torrent in c.torrents_info if torrent['_id'] == _torrent['_id']] if 'in_client' not in torrent: _ids.add(torrent['_id']) @@ -640,7 +631,6 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 如果还是异常退出，可以用 monitor.py，检测到脚本退出后自动运行脚本 """ clients = self.clients if self.client is None else [c.client for c in t_client[1:]] futures = [executor.submit(cl.downloading_torrents_info, self.status_keys) for cl in clients] for future in as_completed(futures): try: @@ -652,19 +642,14 @@ def locate_client(self, torrents: List[Dict[str, Any]]) -> List[Dict[str, Any]]: for hash_id, data in _info.items(): if hash_id == _id: _ids.remove(_id) [to.update({'in_client': True}) for to in torrents if to.get('_id') == _id] if len(_ids) == 0: executor._threads.clear() break # ********** 都查不到，说明种子不在已知客户端 [to.update({'in_client': False}) for to in torrents if '_id' in to and 'in_client' not in to] return torrents def get_info_from_client(self) -> List[Dict[str, Any]]: @@ -1148,10 +1133,8 @@ def check_duplicate(self, data: Dict[str, Any]) -> Union[bool, None]: pro_end_time = pytz.timezone(self.get_tz(soup)).localize(dt).timestamp() else: pro_end_time = time() + 86400 [_torrent.update({'promotion': self.to['promotion'], 'pro_end_time': pro_end_time}) for _torrent in t_client[0].torrents_info if _torrent['tid'] == self.to['tid']] logger.warning(f'Magic for torrent {self.to["tid"]} already existed') return True else: @@ -1512,14 +1495,11 @@ def update_upload(self): tid = int(tr.contents[1].a['href'][15:-6]) for torrent in self.torrents_info: if torrent['tid'] == tid: data = {'upload': tr.contents[6].get_text(' ', ''), 'last_get_time': time()} torrent.update(data) if torrent['uploaded'] != '0 B' and tid == self.to['tid']: self.print(f"Last announce upload of torrent {tid} is {torrent['uploaded']}") [_torrent.update(data) for _torrent in t_client[0].torrents_info if _torrent['tid'] == tid] except Exception as e: logger.error(e) -
Jun 14, 2022 . 1 changed file with 2 additions and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -599,7 +599,8 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: table1 = soup1.find('table', {'width': '90%'}) torrent['date'] = table1.time.attrs.get('title') or table1.time.text for tr1 in table1: if tr1.td.text in ['种子信息', '種子訊息', 'Torrent Info', 'Информация о торренте', 'Torrent Info', 'Информация о торренте']: torrent['_id'] = tr1.tr.contents[-2].contents[1].strip() torrent['last_get_time'] = time() -
Jun 14, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -599,7 +599,7 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: table1 = soup1.find('table', {'width': '90%'}) torrent['date'] = table1.time.attrs.get('title') or table1.time.text for tr1 in table1: if tr1.td.text in ['种子信息', '種子訊息', 'Torrent Info', 'Информация о торренте']: torrent['_id'] = tr1.tr.contents[-2].contents[1].strip() torrent['last_get_time'] = time() -
Jun 14, 2022 . 2 changed files with 6 additions and 6 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -36,7 +36,7 @@ def __init__(self): with open(data_path, 'r', encoding='utf-8') as f: fr = f.read() if fr.startswith('magic_info = '): self.magic_info = eval(fr.lstrip('magic_info = ')) @staticmethod def get_pro(tr): This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -466,11 +466,11 @@ def init(cls): with open(data_path, 'r', encoding='utf-8') as f: for line in f: if line.startswith('mode = '): cls.mode = eval(line.lstrip('mode = ')) if line.startswith('magic_info = '): cls.magic_info = eval(line.lstrip('magic_info = ')) if line.startswith('coefficient = '): cls.coefficient = eval(line.lstrip('coefficient = ')) def __init__(self, client: Union[Deluge, None]): self.client = client @@ -808,7 +808,7 @@ def deta(self) -> int: # 返回种子发布时间与当前的时间差 def get_tz(soup: Tag) -> str: tz_info = soup.find('a', {'href': 'usercp.php?action=tracker#timezone'})['title'] pre_suf = [['时区', '，点击修改。'], ['時區', '，點擊修改。'], ['Current timezone is ', ', click to change.']] return [tz_info[len(pre):][:-len(suf)].strip() for pre, suf in pre_suf if tz_info.startswith(pre)][0] def magic(self): for self.to in self.torrents_info: @@ -1535,7 +1535,7 @@ def update_upload(self): with open(data_path, 'r', encoding='utf-8') as _f2: for _line in _f2: if _line.startswith('torrents_info = '): torrents_info = eval(_line.lstrip('torrents_info = ')) MagicAndLimit.init() local_client = None -
Jun 13, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -683,7 +683,7 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: f1 = 0 # 用来标志是否访问了下载页面，此函数内最多访问一次 for _id, data in info.items(): if data['tracker'] and 'daydream.dmhy.best' in data['tracker']: del data['tracker'] data['_id'] = _id -
Jun 13, 2022 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -1,5 +1,5 @@ # 给下载中的种子放魔法，python3.6 及以上应该能运行 # 依赖：pip3 install PyYAML requests bs4 lxml deluge-client loguru func-timeout paramiko pytz # 支持客户端 deluge，其它的客户端自己去写类吧 # 支持配置多个客户端，可以任意停止和重新运行 -
Jun 13, 2022 . 2 changed files with 5 additions and 7 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -48,7 +48,7 @@ def get_pro(tr): pro_dict_1 = {'free': {'dr': 0.0}, '2up': {'ur': 2.0}, '50pct': {'dr': 0.5}, '30pct': {'dr': 0.3}, 'custom': {}} for img in td.select('img') or []: if not [pro.update(data) for key, data in pro_dict_1.items() if key in img['class'][0]]: pro[{'arrowup': 'ur', 'arrowdown': 'dr'}[img['class'][0]]] = float(img.next_element.text[:-1].replace(',', '.')) for span in td.select('span') or []: [pro.update(data) for key, data in pro_dict.items() if key in (span.get('class') and span['class'][0] or '')] This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -571,6 +571,8 @@ def get_info_from_web(self) -> List[Dict[str, Any]]: # ************ 第二步，和 torrent_info 已有信息合并 for _torrent in self.torrents_info: if torrent['tid'] == _torrent['tid']: if (_torrent.get('pro_end_time') or 0) > time(): torrent['promotion'] = _torrent['promotion'] _torrent.update(torrent) torrent.update(_torrent) break @@ -716,10 +718,6 @@ def get_info_from_client(self) -> List[Dict[str, Any]]: f1 = 1 except Exception as e: logger.exception(e) # ********** 第五步，更新网页后还是查不到，标记 tid 为 -1， # 之后客户端的线程不会对这个种子放魔法，这个种子的魔法会由爬网页的线程施加 @@ -759,7 +757,7 @@ def get_pro(tr: Tag) -> List[Union[int, float]]: pro_dict_1 = {'free': {'dr': 0.0}, '2up': {'ur': 2.0}, '50pct': {'dr': 0.5}, '30pct': {'dr': 0.3}, 'custom': {}} for img in td.select('img') or []: # 图标显示 if not [pro.update(data) for key, data in pro_dict_1.items() if key in img['class'][0]]: pro[{'arrowup': 'ur', 'arrowdown': 'dr'}[img['class'][0]]] = float(img.next_element.text[:-1].replace(',', '.')) for span in td.select('span') or []: # 标记显示 [pro.update(data) for key, data in pro_dict.items() if key in (span.get('class') and span['class'][0] or '')] @@ -1033,7 +1031,7 @@ def expected_cost(self, rule: Dict[str, Any]) -> float: # 估计 uc 消耗量 if 'total_size' in self.to: s = int(self.to['total_size'] / 1024 ** 3) + 1 else: s = int(float(self.to['size'].split(' ')[0].replace(',', '.'))) + 1 ttl = self.deta / 2592000 ttl = 1 if ttl < 1 else ttl ur, dr = float(rule['ur']), float(rule['dr'])
NewerOlder