diff --git a/db/mongo_manager.py b/db/mongo_manager.py index 0c73fb1..518e9d0 100755 --- a/db/mongo_manager.py +++ b/db/mongo_manager.py @@ -87,6 +87,7 @@ class MongoDbManager: collection_to_use = self.db[LINKS_TO_VALIDATE] updated_at = time.strftime("%H:%M:%S", time.localtime()) _ip_country = "FR" + serial = "requests" # find ip_country info for _contact in _all_contact_list: if _contact.mail == mail_address: @@ -96,6 +97,7 @@ class MongoDbManager: collection_to_use.replace_one(filter={'_id': mail_address, }, replacement={ u'url': link, u'email': mail_address, + u'serial': serial, u'ip_country': _ip_country, "updated_at": updated_at }, @@ -103,6 +105,7 @@ class MongoDbManager: else: collection_to_use.replace_one(filter={'_id': link, }, replacement={ u'url': link, + u'serial': serial, u'ip_country': _ip_country, "updated_at": updated_at }, diff --git a/mail/mail_constants.py b/mail/mail_constants.py index 92ad891..bc5441a 100755 --- a/mail/mail_constants.py +++ b/mail/mail_constants.py @@ -86,47 +86,65 @@ def create_imap(login: str): if DOMAIN_163 in login: imap = IMAPClient(IMAP_SERVER_163, use_uid=True) elif DOMAIN_YAHOO in login: - imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) + # imap = imaplib.IMAP4_SSL(YAHOO_IMAP_SERVER) + imap = IMAPClient(YAHOO_IMAP_SERVER, use_uid=True) elif DOMAIN_FIREMAIL_DE in login: # imap = imaplib.IMAP4_SSL(SERVER_FIREMAIL_DE) imap = IMAPClient(SERVER_FIREMAIL_DE, use_uid=True) elif DOMAIN_GMX in login or DOMAIN_GMX_FR in login or DOMAIN_GMX_US in login or DOMAIN_GMX_CH in login or DOMAIN_GMX_PT in login or DOMAIN_GMX_SG in login: - imap = imaplib.IMAP4_SSL(SERVER_GMX) + # imap = imaplib.IMAP4_SSL(SERVER_GMX) + imap = IMAPClient(SERVER_GMX, use_uid=True) elif DOMAIN_SINA in login: # imap = imaplib.IMAP4_SSL(IMAP_SERVER_SINA) imap = IMAPClient(IMAP_SERVER_SINA, use_uid=True) elif DOMAIN_HOTMAIL in login or DOMAIN_OUTLOOK_COM in login: - imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER) + # imap = imaplib.IMAP4_SSL(HOTMAIL_IMAP_SERVER) + imap = IMAPClient(HOTMAIL_IMAP_SERVER, use_uid=True) elif DOMAIN_RAMBLER_RU in login: - imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER) + # imap = imaplib.IMAP4_SSL(RAMBLER_IMAP_SERVER) + imap = IMAPClient(RAMBLER_IMAP_SERVER, use_uid=True) elif DOMAN_BTVM_NE_JP in login: - imap = imaplib.IMAP4_SSL(BTVM_NE_JP) + # imap = imaplib.IMAP4_SSL(BTVM_NE_JP) + imap = IMAPClient(BTVM_NE_JP, use_uid=True) elif DOMAN_GMAIL in login: - imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993) + # imap = imaplib.IMAP4_SSL(SEREVER_GMAIL, port=993) + imap = IMAPClient(SEREVER_GMAIL, use_uid=True) elif DOMAIN_ONET in login: imap = IMAPClient(SERVER_IMAGE_ONET, use_uid=True) elif DOMAIN_TIM_IT in login: - imap = imaplib.IMAP4(TIME_IT_SERVER) + # imap = imaplib.IMAP4(TIME_IT_SERVER) + imap = IMAPClient(TIME_IT_SERVER, use_uid=True) elif DOMAIN_ALICE_IT in login: - imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143) + # imap = imaplib.IMAP4(ALICE_IMAP_SERVER, port=143) + imap = IMAPClient(ALICE_IMAP_SERVER, use_uid=True) elif DOMAIN_MARS_DTI_NE_JP in login: - imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) + # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) + imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True) elif DOMAN_AURORA_DTI_NE_JP in login: - imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) + # imap = imaplib.IMAP4(MARS_DTI_NE_JP_SERVER, port=143) + imap = IMAPClient(MARS_DTI_NE_JP_SERVER, use_uid=True) elif DOMAIN_NAVER in login: - imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993) + # imap = imaplib.IMAP4_SSL(NAVER_SERVER, port=993) + imap = IMAPClient(NAVER_SERVER, use_uid=True) elif DOMAIN_GMX_DE in login or DOMAIN_GMX_NET in login: - imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993) + # imap = imaplib.IMAP4_SSL(SERVER_GMX_NET, port=993) + imap = IMAPClient(SERVER_GMX_NET, use_uid=True) elif DOMAIN_GMX_AT in login: - imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993) + # imap = imaplib.IMAP4_SSL(SERVER_GMX_AT, port=993) + imap = IMAPClient(SERVER_GMX_AT, use_uid=True) elif DOMAIN_GAZETA_PL in login: - imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993) + # imap = imaplib.IMAP4_SSL(IMAP_SERVER_DOMAIN_GAZETA_PL, port=993) + imap = IMAPClient(IMAP_SERVER_DOMAIN_GAZETA_PL, use_uid=True) elif DOMAIN_INBOX_LV in login: - imap = imaplib.IMAP4_SSL(INBOX_LV, port=993) + # imap = imaplib.IMAP4_SSL(INBOX_LV, port=993) + imap = IMAPClient(INBOX_LV, use_uid=True) elif DOMAIN_WEB_DE in login: - imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993) + # imap = imaplib.IMAP4_SSL(SERVER_WEB_DE, port=993) + imap = IMAPClient(SERVER_WEB_DE, use_uid=True) elif DOMAIN_PISS_MAIL in login or DOMAIN_CHILD_PIZZA in login or DOMAIN_DMC_CHAT in login or DOMAIN_GENOCIDE_FUN in login or DOMAIN_HATESJE_WS in login or DOMAIN_INCEL_EMAIL in login or DOMAIN_SHITPOSTING_EXPERT in login: - imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993) + # imap = imaplib.IMAP4_SSL(SERVER_PISS_MAIL, port=993) + imap = IMAPClient(SERVER_PISS_MAIL, use_uid=True) else: - imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) + # imap = imaplib.IMAP4_SSL(AOL_IMAP_SERVER) + imap = IMAPClient(AOL_IMAP_SERVER, use_uid=True) return imap diff --git a/mail/mail_reader_all_contacts.py b/mail/mail_reader_all_contacts.py index 75bc7ac..96cb605 100755 --- a/mail/mail_reader_all_contacts.py +++ b/mail/mail_reader_all_contacts.py @@ -9,6 +9,7 @@ from email.message import Message from imapclient import IMAPClient from db.mongo_manager import MONGO_STORE_MANAGER +from excel_reader import read_contacts from mail.mail_constants import DOMAIN_HOTMAIL, create_imap from models.mail_pojo import MailPojo, MailAddress @@ -179,7 +180,7 @@ class MailReader(): body = body + part.get_payload(decode=True).decode("utf-8") elif part.get_content_type() == "text/plain": body = body + part.get_payload() - if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Validation=20de=20votre=20demande=20de=20rendez-vous" in subject: + if VALIDATION_URL_SUBJECT_fr in subject or VALIDATION_URL_SUBJECT_EN in subject or "Votre=20demande=20de=20rendez-vous" in subject: mail = MailPojo(subject=subject, body=body, from_address=from_address) mail.isImapClient = True print("email is {}".format(self.login)) @@ -262,7 +263,39 @@ def find_links_to_validate_from_mail_list(mail_list: list, logger): if __name__ == '__main__': - mail_address1 = MailAddress(mail="tzfhygwwjiwnf@hotmail.com", password="rqO4Pjm7NQ") - mail_list = [mail_address1] + # mail_address1 = MailAddress(mail="tinagonzales685585@aol.com", password="yhihvdkrbxnksema") + # mail_list = [mail_address1] + contact_to_book_list = read_contacts( + # file_name="/Users/lpan/Desktop/contact_list_2025-01-16_yahoo_100.xlsx") + # file_name="/Users/lpan/Desktop/extracted_yahoo_contacts_129_24_03_win.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_2025-04-02.xlsx") + # file_name="/Users/lpan/Desktop/17_18_04_to_test_win.xlsx") + # file_name="/Users/lpan/Desktop/real_name_contacts_200_aol_win.xlsx") + # file_name="/Users/lpan/Desktop/real_name_contacts_200_aol_mac.xlsx") + # file_name="/Users/lpan/Desktop/real_name_contacts_500_27_03_25_fixed_mac.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_2025-04-05.xlsx") + # file_name="/Users/lpan/Desktop/real_name_contacts_400_aol_mac.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_2025-04-07.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_2025-04-05.xlsx") + # file_name="/Users/lpan/Desktop/real_name_contacts_92_gmx_sg_win.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_not_used_contacts.xlsx") + # file_name="/Users/lpan/Desktop/contact_list_2025-06-07.xlsx") + file_name="/Users/lpan/Desktop/contact_list_2025-06-09_2.xlsx") + all_mail_list = MONGO_STORE_MANAGER.get_destination_emails() + mail_list_to_check = [] + for contact in contact_to_book_list: + for mail in all_mail_list: + if contact.mail == mail.mail: + mail_list_to_check.append(mail) logger = logging.getLogger() - find_links_to_validate_from_mail_list(mail_list, logger) + _all_links = MONGO_STORE_MANAGER.get_links_to_validate() + filter_mail = [] + for mail_pojo in mail_list_to_check: + _to_add = True + for _link in _all_links: + if _link.email == mail_pojo.mail: + _to_add = False + if _to_add: + filter_mail.append(mail_pojo) + + find_links_to_validate_from_mail_list(filter_mail, logger) diff --git a/migrate_to_data_cache_2.py b/migrate_to_data_cache_2.py index 6ee4549..e9fd69d 100644 --- a/migrate_to_data_cache_2.py +++ b/migrate_to_data_cache_2.py @@ -3,6 +3,6 @@ from queue_message.CookiesPublisher import MORNING_DATA_CACHE_2, MORNING_DATA_CA from workers.MessagerTransporter import migrate_message_to_queue, empty_message_just_to if __name__ == '__main__': - migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK, to_queue=MORNING_DATA_CACHE) + migrate_message_to_queue(from_queue=MORNING_DATA_CACHE, to_queue=MORNING_DATA_CACHE_BAK) # empty_message_just_to(2000, queue_name=MORNING_DATA_CACHE_BAK) # migrate_message_to_queue(from_queue=MORNING_DATA_CACHE_BAK,to_queue=REQUEST_DATA_QUEUE_DE) diff --git a/models/LinkPojo.py b/models/LinkPojo.py index 6a0b235..7f75958 100644 --- a/models/LinkPojo.py +++ b/models/LinkPojo.py @@ -11,7 +11,9 @@ class LinkPojo(): def from_firestore_dict(source): updated_at = source['updated_at'] email = source['email'] - model = source['model'] + model = "" + if 'model' in source: + model = source['model'] serial = source['serial'] url = source['url'] ip_country = "FR" diff --git a/queue_message/appointmentrequestsender.py b/queue_message/appointmentrequestsender.py index 5b95a2f..4492d50 100644 --- a/queue_message/appointmentrequestsender.py +++ b/queue_message/appointmentrequestsender.py @@ -239,6 +239,7 @@ class AppointmentRequestSender(threading.Thread): self.channel.start_consuming() def retrieve_invalidate_urls(self): + self.list_to_retrieve_mails = self.initial_contact_list # 如果没有已读邮件，而且需要读邮件的联系人表不为空，就读取未读邮件 if not self.already_read_emails and len(self.list_to_retrieve_mails) > 0: self.logger.info("will retrieve validate urls") diff --git a/request_sender.py b/request_sender.py index 2826c30..d053351 100644 --- a/request_sender.py +++ b/request_sender.py @@ -71,10 +71,10 @@ def send_appointment_request(message_queue_name, _contact_list): def start_send_requests(): print("start send requests") - contacts_file_path = '~/Desktop/11_05_to_test.xlsx' - _contact_list = read_contacts(contacts_file_path)[:-1] + contacts_file_path = '~/Desktop/06_05_to_test.xlsx' + _contact_list = read_contacts(contacts_file_path)[:1] _contact_list_to_book = filter_contacts(_contact_list) - _segment_number = 5 + _segment_number = 1 logger.info("{} contacts to book".format(len(_contact_list_to_book))) last_thread = None for i in range(0, _segment_number): diff --git a/request_sender_test.py b/request_sender_test.py index 545b4d9..3f423e0 100644 --- a/request_sender_test.py +++ b/request_sender_test.py @@ -103,6 +103,6 @@ if __name__ == '__main__': # '~/Desktop/contact_list_2024-05-21.xlsx', # '~/Desktop/15_05_to_test.xlsx'] # file_list = ['~/Desktop/15_05_to_test.xlsx', '~/Desktop/16_05_to_test.xlsx'] - file_list = ['~/Desktop/contact_list_2024-10-05_gmx_net_100.xlsx'] - send_request_for_file_list(file_list=file_list, thread_number=1, + file_list = ['~/Desktop/contact_list_2025-06-09_2.xlsx'] + send_request_for_file_list(file_list=file_list, thread_number=10, data_queue_name=MORNING_DATA_CACHE, stop_at_hour=19, stop_at_mins=50) diff --git a/workers/link_validator_with_provided_list.py b/workers/link_validator_with_provided_list.py index bae1f1e..4ddc44c 100644 --- a/workers/link_validator_with_provided_list.py +++ b/workers/link_validator_with_provided_list.py @@ -28,7 +28,7 @@ logger = logging.getLogger() def filter_link_pojo_list_with_serial(_received_dict, link_to_validate_list): _serial = _received_dict["serial"] _model = _received_dict["model"] - _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "bitbrowser", link_to_validate_list)) + _bit_browser_list = list(filter(lambda link_pojo: link_pojo.serial == "requests", link_to_validate_list)) if len(_bit_browser_list) > 0: return _bit_browser_list else: @@ -210,12 +210,18 @@ def validate_links(cookiesPublisher, queue_name: str, link_list: list, _contact_ def validate_all_links(_contact_serial_list): print("will get all links") all_link_list = MONGO_STORE_MANAGER.get_links_to_validate() + link_to_validated = [] + for _link in all_link_list: + # print("serial is "+_link.serial) + print("email is "+_link.email) + if _link.serial == "requests": + link_to_validated.append(_link) # get the first 50 links - if len(all_link_list) == 0: + if len(link_to_validated) == 0: return - # divided = 1 + divided = 2 # default_segment_number = 20 - # _first_25_percent_links = all_link_list[0:(int(len(all_link_list) / divided))] + _first_25_percent_links = link_to_validated[0:(int(len(all_link_list) / divided))] _first_25_percent_links = all_link_list _queue_name = MORNING_DATA_CACHE # _queue_name = MORNING_DATA_CACHE_BAK diff --git a/workers/sender.py b/workers/sender.py index 7965f57..f5b0236 100644 --- a/workers/sender.py +++ b/workers/sender.py @@ -160,7 +160,7 @@ class Sender: headers['sec-ch-ua-full-version-list'] = _chrome_ua_version data = {'check': '', '_csrf': _csrf, 'prefer': _selected_store, 'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(), - 'phone_country': "FR", 'phone_number': "+33 " + contact.phone, 'email': contact.mail, + 'phone_country': "FR", 'phone_number': "0" + contact.phone, 'email': contact.mail, 'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'} self.logger.info(data) self.logger.info("try to request for mail:{}".format(contact.mail))