first commit

db/mongo_manager.py

@@ -0,0 +1,74 @@
+import datetime
+import logging
+
+from pymongo import MongoClient
+
+from models.ReserveResultPojo import ReserveResultPojo
+from models.contact_pojo import ContactPojo
+
+MONGO_DB_URL = "mongo.lpaconsulting.fr"
+CAPTCHA_ERROR_COLLECTION_PREFIX = "CAPTCHA_ERROR_"
+BLACK_LIST = "BLACK_LIST"
+ACCEPTED_APPOINTMENT_LIST = "ACCEPTED_APPOINTMENT_LIST"
+CONTACT_LIST_TO_BOOK = "CONTACT_LIST_TO_BOOK"
+EMAIL_LIST = "EMAIL_LIST"
+DESTINATION_EMAIL_LIST = "DESTINATION_EMAIL_LIST"
+LINKS_TO_VALIDATE = "LINKS_TO_VALIDATE"
+INVALID_EMAIL_LIST = "INVALID_EMAIL_LIST"
+
+
+class MongoDbManager:
+    def __init__(self):
+        client = MongoClient(MONGO_DB_URL, username='appointment', password='Rdv@2022', authSource='appointment')
+        self.db = client.appointment
+        self.logger = logging.getLogger("mongoDb")
+
+    def insert_one(self, collection_name: str, dict: dict):
+        collection_to_use = self.db[collection_name]
+        collection_to_use.insert_one(dict)
+
+    def insert_reserve_result(self, collection_name, reserve: ReserveResultPojo):
+        try:
+            collection_to_use = self.db[collection_name]
+            collection_to_use.replace_one(filter={'_id': reserve.id, }, replacement=reserve.to_firestore_dict(),
+                                          upsert=True)
+        except Exception as Error:
+            self.logger.info(Error)
+
+    def get_all_successful_items_for_day(self) -> list:
+        collection_name = str(datetime.date.today())
+        result_list = []
+        cursor = self.db[collection_name]
+        for document in cursor.find():
+            result_list.append(ReserveResultPojo.from_firestore_dict(document))
+        return result_list
+
+    def count_all_successful_items_for_day(self, day: str) -> list:
+        return self.db[day].count_documents({})
+
+    def get_all_successful_items_for_yesterday(self) -> list:
+        yesterday = datetime.date.today() - datetime.timedelta(days=1)
+        collection_name = str(yesterday)
+        result_list = []
+        cursor = self.db[collection_name]
+        for document in cursor.find():
+            result_list.append(ReserveResultPojo.from_firestore_dict(document))
+        return result_list
+
+    def get_all_successful_items_for_one_day(self, day_in_str: str) -> list:
+        collection_name = day_in_str
+        result_list = []
+        cursor = self.db[collection_name]
+        for document in cursor.find():
+            result_list.append(ReserveResultPojo.from_firestore_dict(document))
+        return result_list
+
+    def get_all_contact_to_book_list(self) -> list:
+        result_list = []
+        cursor = self.db[CONTACT_LIST_TO_BOOK]
+        for document in cursor.find():
+            result_list.append(ContactPojo.from_firestore_dict(document))
+        return result_list
+
+
+MONGO_STORE_MANAGER = MongoDbManager()

excel_reader.py

@@ -0,0 +1,127 @@
+import json
+import random
+import string
+
+import pandas as pandas
+
+from models.contact_pojo import ContactPojo
+
+phone_number_prefix = ['7']
+chinnese_number_prefix = ['13', '15', '18']
+
+
+def read_contacts(file_name) -> list:
+    print("read file " + file_name)
+    contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records')
+    contact_dict_list = json.loads(contact_list_in_json)
+    contact_list = []
+    for contact_dict in contact_dict_list:
+        if contact_dict['name']:
+            raw_name = contact_dict['name'].strip()
+            name = raw_name.split(' ')
+            last_name = name[0]
+            if len(name) == 2:
+                first_name = name[-1]
+            else:
+                first_name = ''.join(name[1:len(name)])
+
+            contact = ContactPojo(phone_number=contact_dict['phone'],
+                                  last_name=last_name,
+                                  first_name=first_name,
+                                  passport_number=contact_dict['passport'],
+                                  mail=contact_dict['email'])
+            contact_list.append(contact)
+    return contact_list
+
+
+class ExcelHelper:
+
+    def __init__(self):
+        self._df = pandas.Series()
+
+    def check_contact_list(self, file_name):
+        contact_list = read_contacts(file_name)
+        for contact in contact_list:
+            if contact.first_name is None or len(contact.first_name) == 0:
+                print("error in firstName for " + contact.mail)
+            if contact.last_name is None or len(contact.last_name) == 0:
+                print("error in last_name for " + contact.mail)
+            if contact.phone is None or len(contact.phone) == 0:
+                print("error in phone for " + contact.mail)
+            if contact.passport is None or len(contact.passport) == 0:
+                print("error in passport_number for " + contact.mail)
+            if contact.mail is None or len(contact.mail) == 0:
+                print("error in mail for " + contact.phone_number)
+
+
+    def read_names(self, file_name) -> list:
+        contact_list_in_json = pandas.read_excel(file_name).to_json(orient='records')
+        contact_dict_list = json.loads(contact_list_in_json)
+        contact_list = []
+        count = 2
+        for contact_dict in contact_dict_list:
+            if contact_dict['name']:
+                raw_name = contact_dict['name'].strip()
+                name = raw_name.split(' ')
+                if len(name) == 1:
+                    name = raw_name.split('\xa0')
+                if len(name) == 1:
+                    print("error in " + str(name))
+                last_name = name[0]
+                if len(name) == 2:
+                    first_name = name[-1]
+                else:
+                    first_name = ''.join(name[1:len(name)])
+
+                contact = ContactPojo(phone_number="",
+                                      last_name=last_name,
+                                      first_name=first_name,
+                                      passport_number="",
+                                      mail="")
+
+                if len(first_name) == 0:
+                    print("first_name is empty: position:" + str(count))
+                    print(name)
+                if len(last_name) == 0:
+                    print("last_name is empty: position:" + str(count))
+                count = count + 1
+                contact_list.append(contact)
+
+        return contact_list
+
+
+def get_random_fr_phone_numbers():
+    length = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    id_number = random.choice(phone_number_prefix) + str(ran)
+    return id_number
+
+
+def get_random_cn_phone_numbers():
+    length = 8  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    id_number = random.choice(phone_number_prefix) + str(ran)
+    prefix = random.choice(chinnese_number_prefix)
+    return prefix + id_number
+
+
+def generate_email_from_name(first_name: str, last_name: str) -> str:
+    length = 2  # number of characters in the string.
+    ran = ''.join(random.choices(string.digits, k=length))
+    separator = ['.', '_', '']
+    domains = ['gmail.com', 'hotmail.com', 'yahoo.com', 'aol.com', 'outlook.com', 'hotmail.fr', 'gmx.com',
+               'hotmail.com', 'yahoo.com', 'aol.com', 'hotmail.com']
+    email = "{}{}{}{}@{}".format(last_name.lower(), random.choice(separator),
+                                 first_name.replace("-", "").replace("'", "").lower(), ran,
+                                 random.choice(domains))
+    print(email)
+    return email
+
+
+def get_random_id_number() -> str:
+    # write_the_valid_profiles_to_excel()
+    S = 8  # number of characters in the string.
+    # call random.choices() string module to find the string in Uppercase + numeric data.
+    ran = ''.join(random.choices(string.digits, k=S))
+    print("The randomly generated string is : 94" + str(ran))  # print the random data
+    return ran

models/ReserveResultPojo.py

@@ -0,0 +1,131 @@
+import socket
+from dataclasses import dataclass
+from enum import Enum
+from typing import Union
+
+from dataclasses_json import dataclass_json
+
+
+class PublishType(Enum):
+    SUCCESS = "SUCCESS"
+    ERROR = "ERROR"
+    PENDING = "PENDING"
+    DUPLICATED = "DUPLICATED"
+
+
+@dataclass_json
+@dataclass
+class ReserveResultPojo:
+    type: PublishType = PublishType.ERROR
+    phone: str = ""
+    message: str = ""
+    url: str = ""
+    firstName: Union[None, str] = ""
+    lastName: Union[None, str] = ""
+    email: Union[None, str] = ""
+    id: str = ""
+    accepted = None
+    passport: str = ""
+    slot_position = None
+    sim_position = None
+    ccid: str = ""
+    source_from: str = socket.gethostname()
+    store_type = 0
+    url_validated = None
+    created_at = None
+    validated_at = None
+
+    @staticmethod
+    def from_firestore_dict(source):
+        publish_type = PublishType.ERROR
+        if 'type' in source:
+            publish_type = source['type']
+            if publish_type:
+                publish_type = PublishType[publish_type]
+        if 'phone' in source:
+            phone = source['phone']
+        else:
+            phone = ""
+        if 'url' in source:
+            url = source['url']
+        else:
+            url = ""
+        if 'id' in source:
+            id = source['id']
+        if '_id' in source:
+            id = source['_id']
+        else:
+            id = ""
+        if 'email' in source:
+            email = source['email']
+        else:
+            email = ""
+        if 'lastName' in source:
+            lastName = source['lastName']
+        else:
+            lastName = ""
+
+        if 'firstName' in source:
+            firstName = source['firstName']
+        else:
+            firstName = ""
+
+        result = ReserveResultPojo(type=publish_type, phone=phone,
+                                   url=url, email=email,
+                                   firstName=firstName, lastName=lastName)
+        if 'accepted' in source:
+            accepted = source['accepted']
+            result.accepted = accepted
+        if 'message' in source:
+            message = source['message']
+            result.message = message
+        if 'source' in source:
+            source_from = source['source']
+            result.source_from = source_from
+        if 'sim_position' in source:
+            sim_position = source['sim_position']
+            result.sim_position = sim_position
+        if 'slot_position' in source:
+            slot_position = source['slot_position']
+            result.slot_position = slot_position
+        if 'passport' in source:
+            passport = source['passport']
+            result.passport = passport
+        if 'ccid' in source:
+            ccid = source['ccid']
+            result.ccid = ccid
+        if 'store_type' in source:
+            store_type = source['store_type']
+            result.store_type = store_type
+        if 'url_validated' in source:
+            url_validated = source['url_validated']
+            result.url_validated = bool(url_validated)
+        if 'created_at' in source:
+            created_at = source['created_at']
+            result.created_at = created_at
+        if 'validated_at' in source:
+            validated_at = source['validated_at']
+            result.validated_at = validated_at
+        result.id = id
+        return result
+
+    def to_firestore_dict(self):
+        dest = {
+            u'type': self.type.value,
+            u'id': self.id,
+            u'phone': self.phone,
+            u'firstName': self.firstName,
+            u'lastName': self.lastName,
+            u'email': self.email,
+            u'passport': self.passport,
+            u'url': self.url,
+            u'sim_position': self.sim_position,
+            u'slot_position': self.slot_position,
+            u'source_from': self.source_from,
+            u'created_at': self.created_at,
+            u'store_type': self.store_type,
+            u'accepted': self.accepted,
+            u'url_validated': self.url_validated,
+        }
+
+        return dest

models/contact_pojo.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ContactPojo:
+    phone: str
+    passport: str
+    last_name: str
+    first_name: str
+    mail: str
+    ccid: str
+    position: int
+    note: str
+
+    def __init__(self, phone_number: str, passport_number: str, last_name: str, first_name: str, mail: str,
+                 ccid: str = "",
+                 position: int = 0):
+        self.phone = str(phone_number).replace(".0", "")
+        self.passport = passport_number
+        self.last_name = last_name
+        self.first_name = first_name
+        self.ccid = ccid
+        self.mail = mail
+        self.position = position
+        self.note = ""
+
+    def to_firestore_dict(self):
+        dest = {
+            u'phone': self.phone,
+            u'passport': self.passport,
+            u'last_name': self.last_name,
+            u'first_name': self.first_name,
+            u'mail': self.mail,
+            u'ccid': self.ccid,
+            u'position': self.position
+        }
+
+        return dest
+
+    @staticmethod
+    def from_firestore_dict(source):
+        ccid = source['ccid']
+        phone = source['phone']
+        position = source['position']
+        passport = source['passport']
+        email = source['mail']
+        last_name = source['last_name']
+        first_name = source['first_name']
+        result = ContactPojo(ccid=ccid, phone_number=phone, passport_number=passport, position=position, mail=email,
+                             last_name=last_name, first_name=first_name)
+        return result

request_sender.py

@@ -0,0 +1,103 @@
+import datetime
+import random
+import time
+from http.cookies import SimpleCookie
+
+import requests
+
+from db.mongo_manager import MONGO_STORE_MANAGER
+from excel_reader import read_contacts
+from models.ReserveResultPojo import ReserveResultPojo, PublishType
+from models.contact_pojo import ContactPojo
+
+FR_PROXY_MOBILE = {
+    'http': 'http://7039778-mobile-country-FR:2ji8e3r0nh@89.39.106.148:14806',
+    'https': 'http://7039778-mobile-country-FR:2ji8e3r0nh@89.39.106.148:14806'
+}
+
+IPFIY = 'http://api.ipify.org'
+HERMES_REGISTER = "https://rendezvousparis.hermes.com/client/register"
+NGROK_TEST = "https://bcc6-193-164-156-53.ngrok-free.app"
+
+
+def is_already_sent(contact: ContactPojo) -> bool:
+    already_sent_contacts = MONGO_STORE_MANAGER.get_all_successful_items_for_day()
+    for required_contact in already_sent_contacts:
+        if contact.mail == required_contact.email:
+            return True
+    return False
+
+
+class Sender:
+
+    def __init__(self):
+        self.store_type = "random"
+        self.cookie = SimpleCookie()
+        self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
+        self.cookie.load(self.cookie_str)
+
+    def publish_message_to_queue(self, contact: ContactPojo, status: PublishType, url: str):
+        # create the message
+        if url == "https://rendezvousparis.hermes.com/client/welcome":
+            return
+        id = url.split("/")[-1]
+        result = ReserveResultPojo(type=status, phone=contact.phone, message=status.value, url=url,
+                                   firstName=contact.first_name, lastName=contact.last_name, email=contact.mail,
+                                   passport=contact.passport, ccid=contact.ccid)
+        result.id = id
+        result.store_type = self.store_type
+        result.created_at = time.strftime("%H:%M:%S", time.localtime())
+        collection_name = str(datetime.date.today())
+        MONGO_STORE_MANAGER.insert_reserve_result(collection_name=collection_name, reserve=result)
+
+    def send_request_for_list(self, contact_list: list):
+        pass
+
+    def send_request(self, url, contact: ContactPojo):
+        headers = {'Content-Type': 'application/x-www-form-urlencoded',
+                   'User-Agent': 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Mobile Safari/537.36',
+                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+                   'Cookie': self.cookie_str, 'Referer': 'https://rendezvousparis.hermes.com/client/register',
+                   'Sec-Fetch-Mode': 'navigate',
+                   'Sec-Fetch-Dest': 'document',
+                   'Accept-Language': 'fr-FR,fr;q=0.6'}
+        data = {'check': '', '_csrf': 'SqI0z6xt-guBLrwEQLcqhTczFWh_EpcffX9w', 'prefer': '',
+                'surname': contact.last_name.capitalize(), 'name': contact.first_name.capitalize(),
+                'phone_country': "FR", 'phone_number': contact.phone, 'email': contact.mail,
+                'passport_id': contact.passport, 'processing': 'on', 'cgu': 'on'}
+        print(data)
+        response = requests.post(url=url, proxies=FR_PROXY_MOBILE, verify=False, headers=headers, data=data)
+        print(response.status_code)
+        if response.status_code == 200:
+            # add to mongodb
+            print(response.text)
+            print(response.url)
+            self.publish_message_to_queue(contact, status=PublishType.SUCCESS, url=response.url)
+            cookies_to_set = response.headers['set-cookie']
+            self.cookie.load(cookies_to_set)
+            new_cookies = {k: v.value for k, v in self.cookie.items()}
+            new_coolies_str = ""
+            for key in new_cookies:
+                new_coolies_str = new_coolies_str + key + "=" + new_cookies[key] + ";"
+            print(new_coolies_str)
+            self.cookie_str = new_coolies_str
+            return True
+        else:
+            return False
+
+
+if __name__ == '__main__':
+    sender = Sender()
+    contact_list = read_contacts('/Users/panlei/Desktop/yahoo_aol_valid_26-1.xlsx')
+    sub_contact_list = contact_list[200:300]
+    for con in sub_contact_list:
+        if not is_already_sent(con):
+            time.sleep(random.randint(2, 10))
+            can_continue = sender.send_request(HERMES_REGISTER, con)
+            if not can_continue:
+                print("cannot continue")
+                break
+        else:
+            print(con.mail + "--> skip")
+        print(con.mail)
+    # sender.send_request(HERMES_REGISTER, contact_list[-5])