support ip_country field

workers/link_validator.py

@@ -16,7 +16,8 @@ from workers.proxies_constants import PROXY_LIST_FR
 class LinkValidator(threading.Thread):
 
     def __init__(self, link_to_validate_list: list,
-                 cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR"):
+                 cookiesPublisher: CookiesPublisher, proxy_to_use, queue_to_listen=REQUEST_DATA_QUEUE, ip_country="FR",
+                 limit=40):
         super().__init__()
         self.cookie = SimpleCookie()
         self.cookiesPublisher = cookiesPublisher
@@ -25,6 +26,7 @@ class LinkValidator(threading.Thread):
         self.ip_country = ip_country
         self.filter_with_ip_country()
         self.proxy_to_use = proxy_to_use
+        self.limit = limit
         # self.cookie_str = 'datadome=~pxdHFAvsQl2rvDrTzhPgCHxu~4TBcePTTE~Cy8Rgol6oMRc11gA02VRp0Z3uEDUszCjacubNu7vbfQCh27gz8RC10u_325pt_gsMmJh1ScGvOofVJiVAbEKvSEUjd82;policy=accepted;app.sig=PhjmDkq_dI49pADppDNKxpLe_G4;app=eyJmbGFzaCI6e30sImNhY2hlZmxhc2giOltdLCJjc3JmU2VjcmV0IjoiYnRodHNYU1lvdnl4RzVGakpGRDZsQ0JtIn0=;lang=fr;'
 
     def set_up_connection(self):
@@ -85,11 +87,13 @@ class LinkValidator(threading.Thread):
 
     def on_message(self, ch, method, properties, body):
         print(f" [x] Received {body}")
+        _message_in_queue_count = self.cookiesPublisher.message_count()
+        print("message count in queue is {}".format(_message_in_queue_count))
         self.link_to_validate_list = MONGO_STORE_MANAGER.get_links_to_validate()
         self.filter_with_ip_country()
         self.cookie_str = body.decode("UTF-8")
         random.shuffle(self.link_to_validate_list)
-        if len(self.link_to_validate_list) > 0:
+        if len(self.link_to_validate_list) > 0 and _message_in_queue_count > self.limit:
             print("links number is {}".format(len(self.link_to_validate_list)))
             for con in self.link_to_validate_list:
                 # if not is_already_sent(con):
@@ -117,7 +121,7 @@ class LinkValidator(threading.Thread):
 
 
 def validate_with_FR_ip():
-    _queue_name = REQUEST_DATA_QUEUE_TEST
+    _queue_name = REQUEST_DATA_QUEUE
     link_list = MONGO_STORE_MANAGER.get_links_to_validate()
     cookiesPublisher = CookiesPublisher(queue_name=_queue_name)
     cookiesPublisher.set_up_connection()
@@ -131,7 +135,7 @@ def validate_with_FR_ip():
     _fr_proxy_to_use = random.choice(PROXY_LIST_FR)
     receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
                              proxy_to_use=_fr_proxy_to_use,
-                             queue_to_listen=_queue_name, ip_country="FR")
+                             queue_to_listen=_queue_name, ip_country="FR", limit=10)
     receiver.set_up_connection()
     receiver.listen_to_queue(receiver.on_message)
     pass

workers/validate_links_DE.py

@@ -23,7 +23,7 @@ def validate_with_DE_ip():
     _de_proxy_to_use = random.choice(PROXY_LIST_DE)
     receiver = LinkValidator(link_to_validate_list=_link_list_to_click, cookiesPublisher=cookiesPublisher,
                              proxy_to_use=_de_proxy_to_use,
-                             queue_to_listen=_queue_name, ip_country="DE")
+                             queue_to_listen=_queue_name, ip_country="DE", limit=0)
     receiver.set_up_connection()
     receiver.listen_to_queue(receiver.on_message)
     pass