import logging import sys from concurrent.futures import ThreadPoolExecutor from src import params, definitions from src.db.mongo_manager import MONGO_STORE_MANAGER from src.logs.AppLogging import init_logger from src.pojo.ModeEnum import ModeEnum from src.pojo.contact_pojo import ContactPojo from src.proxy.proxy_type import ProxyType from src.utils.black_list_checker import can_send_request from src.utils.excel_reader import ExcelHelper from src.workers.commandor_page import CommandorPage # used to save the current slot position init_logger() logger = logging.getLogger() logger.addHandler(logging.StreamHandler(stream=sys.stdout)) def start_book(start_number, end_number, store_choose_state=0, max_workers=20, proxy_type=ProxyType.BRIGHT_DATA, mode: ModeEnum = ModeEnum.MANUAL, headless=False): # read the contact, and contact the 2 objects together excel_reader = ExcelHelper() all_contacts = excel_reader.read_contacts() if len(all_contacts) <= end_number: end_number = len(all_contacts) contacts = all_contacts[start_number - 1: end_number] with ThreadPoolExecutor(max_workers=max_workers) as executor: for contact in contacts: proxy = get_proxy(proxy_type) # start the task in thread if can_send_request(contact): executor.submit( CommandorPage(contact, store_type=store_choose_state, proxy_type=proxy_type, mode=mode, headless=headless).start_page, proxy) else: logger.info("do not send request --> skip") def recheck_the_captcha_error_contacts(on_no_contact_found, store_type=0, mode: ModeEnum = ModeEnum.MANUAL, max_workers=20): # get all the contacts in captcha error contact_list = MONGO_STORE_MANAGER.get_captcha_error_contacts_for_current_day() if len(contact_list) == 0: on_no_contact_found() with ThreadPoolExecutor(max_workers=max_workers) as executor: for errorContact in contact_list: contact = ContactPojo.get_contact_from_error_contact(errorContact) proxy = get_proxy() # start the task in thread executor.submit( CommandorPage(contact, store_type=store_type, mode=mode).start_page, proxy) def get_proxy(proxy_type=ProxyType.BRIGHT_DATA): return params.get_proxy(proxy_type) if __name__ == '__main__': # 修改联系人行,结束联系人行 第三个参数store等于0的时候是随机,传入1的时候是总店 start_book(901, 1200, store_choose_state=1, mode=ModeEnum.AUTOMATIC, headless=True) # recheck_the_captcha_error_contacts(store_type=1, mode=ModeEnum.AUTOMATIC, on_no_contact_found=lambda: None)