Add support for timeout and retry, and do some error handling

Update README

README.md

@@ -13,7 +13,7 @@ After repeated attempts to validate email addresses, Office 365 will flag these
 leading to false positives. This program will therefore search for the "ThrottleStatus" parameter
 in order to decide if the response can be considered as valid or not. When the parameter is set to
 1, this means that the server throttled our request. In this case, and if we are using Tor, the
-program will try to regenerate the tor circuit and retry the previous request. This generally
+program will try to regenerate the tor circuit and retry the last request. This generally
 allows us to get reliable results.
 
 Furthermore, this program will also make requests with random user agents in order to be even more
@@ -52,8 +52,8 @@ output throttled ones to a different file with the `--output-fail` parameter, in
 to retry them later.
 
 ```
-usage: o365creeper-ng.py [-h] (-e EMAIL | -f FILE) [-o OUTPUT] [--output-fail OUTPUT_FAIL] 
-			[-t] [-p SOCKS_PORT] [-c CONTROL_PORT] [-s CONTROL_PW]
+usage: o365creeper-ng.py [-h] (-e EMAIL | -f FILE) [-o OUTPUT] [--output-fail OUTPUT_FAIL]
+			[-t] [-p SOCKS_PORT] [-c CONTROL_PORT] [-s CONTROL_PW] [--timeout TIME] [--retry N]
 
 Enumerates valid email addresses from Office 365 without submitting login attempts.
 
@@ -73,6 +73,8 @@ optional arguments:
                         Tor control port to use (default: 9051).
   -s CONTROL_PW, --tor-control-pw CONTROL_PW
                         Password for Tor control port (default: None).
+  --timeout TIME        Stop waiting for a response after TIME seconds (default: 60).
+  --retry N             Retry up to N times in case of error (default: 3).
 ```
 
 ### Examples:

o365creeper-ng.py

@@ -22,6 +22,7 @@
 import time
 from fake_useragent import UserAgent
 from pathlib import Path
+from requests.exceptions import ConnectionError, Timeout
 from stem import Signal
 from stem.control import Controller
 from typing import Dict
@@ -83,6 +84,22 @@
         type=str,
         help='Password for Tor control port (default: %(default)s).',
         )
+parser.add_argument(
+        '--timeout',
+        default=60,
+        type=int,
+        metavar='TIME',
+        help=('Stop waiting for a response after %(metavar)s ' +
+            'seconds (default: %(default)s).'),
+        )
+parser.add_argument(
+        '--retry',
+        default=3,
+        type=int,
+        metavar='N',
+        help=('Retry up to %(metavar)s times in case of error ' +
+            '(default: %(default)s).'),
+        )
 
 args = parser.parse_args()
 
@@ -98,7 +115,9 @@
         'output': args.output,
         'output_fail': args.output_fail,
     },
-    'email': args.email
+    'email': args.email,
+    'timeout': args.timeout,
+    'retry': args.retry,
 }
 
 url = 'https://login.microsoftonline.com/common/GetCredentialType'
@@ -108,6 +127,7 @@ def check_email(
         url: str,
         email: str,
         tor_config: Dict,
+        timeout: int,
         ) -> Dict:
     """
         Check if a given email exists at O365
@@ -122,14 +142,70 @@ def check_email(
                 }
     headers = {'User-Agent': UserAgent().random}
     payload = {'Username': email}
-    r = req.post(url, proxies=proxies, headers=headers, json=payload)
-
-    ret['valid'] = re.search('"IfExistsResult":0,', r.text) is not None
-    ret['throttle'] = re.search('"ThrottleStatus":1,', r.text) is not None
+    try:
+        r = req.post(url, proxies=proxies, headers=headers, json=payload)
+    except ConnectionError:
+        #print(f'{email} - CONNECTION ERROR')
+        ret['valid'] = False
+        ret['throttle'] = False
+        ret['error'] = True
+        ret['exception'] = ConnectionError
+    except Timeout:
+        #print(f'{email} - TIMEOUT')
+        ret['valid'] = False
+        ret['throttle'] = False
+        ret['error'] = True
+        ret['exception'] = Timeout
+    else:
+        ret['valid'] = re.search('"IfExistsResult":0,', r.text) is not None
+        ret['throttle'] = re.search('"ThrottleStatus":1,', r.text) is not None
+        ret['error'] = False
+        ret['exception'] = None
 
     return ret
 
 
+def need_retry(status: dict) -> bool:
+    return status['throttle'] or status['error']
+
+
+def retry(config: Dict) -> None:
+    n = config['retry']
+    while (n > 0):
+        # generate new circuit
+        if config['tor']['use']:
+            with Controller.from_port(
+                    port = config['tor']['control_port']) as c:
+                c.authenticate(password=config['tor']['control_pw'])
+                # TODO: validates auth and try other methods
+                c.signal(Signal.NEWNYM)
+
+        new_check = check_email(url, email, config['tor'], config['timeout'])
+        if need_retry(new_check):
+            n -= 1
+        else:
+            break
+    # still throttling or error occurring :(
+    if need_retry(new_check):
+        if new_check['throttle']:
+            print(f'{email} - THROTTLED')
+        else:
+            print(f'{email} - {new_check["exception"]}')
+        if config['files']['output_fail'] is not None:
+            with config['files']['output_fail'].open(mode='a') as fail_file:
+                fail_file.write(email+'\n')
+    # didn't throttle this time (nor error)
+    else:
+        # is valid email?
+        if new_check['valid']:
+            print(f'{email} - VALID')
+            if config['files']['output'] is not None:
+                with config['files']['output'].open(mode='a') as output_file:
+                    output_file.write(email+'\n')
+        else:
+            print(f'{email} - INVALID')
+
+
 def validate_result(
         check_res: Dict,
         email: str,
@@ -139,47 +215,9 @@ def validate_result(
     """
         Validate results and redo if necessary
     """
-    # is endpoint throttling requests?
-    if throttle := check_res['throttle']:
-        # if using tor, try new circuit(s)
-        if config['tor']['use']:
-            retry = 3
-            while (retry > 0):
-                with Controller.from_port(
-                        port = config['tor']['control_port']) as c:
-                    c.authenticate(password=config['tor']['control_pw'])
-                    # TODO: validates auth and try other methods
-                    c.signal(Signal.NEWNYM)
-                new_check = check_email(url, email, config['tor'])
-                if new_check['throttle']:
-                    retry -= 1
-                else:
-                    break
-            # still throttling :(
-            if new_check['throttle']:
-                print(f'{email} - THROTTLED')
-                if config['files']['output_fail'] is not None:
-                    with config['files']['output_fail'].open(mode='a') as fail_file:
-                        fail_file.write(email+'\n')
-            # didn't throttle this time
-            else:
-                # is valid email?
-                if new_check['valid']:
-                    print(f'{email} - VALID')
-                    if config['files']['output'] is not None:
-                        with config['files']['output'].open(mode='a') as output_file:
-                            output_file.write(email+'\n')
-                else:
-                    print(f'{email} - INVALID')
-
-        # not using tor
-        # TODO: try other bypass methods
-        else:
-            print(f'{email} - THROTTLED')
-            if config['files']['output_fail'] is not None:
-                with config['files']['output_fail'].open(mode='a') as fail_file:
-                    fail_file.write(email+'\n')
-
+    # is endpoint throttling requests or some error occured?
+    if need_retry(check_res):
+        retry(config)
     # response was not throttled
     else:
         # is valid email?
@@ -199,12 +237,12 @@ def main():
         with config['files']['input'].open() as file:
             for line in file:
                 email = line.strip()
-                checked = check_email(url, email, config['tor'])
+                checked = check_email(url, email, config['tor'], config['timeout'])
                 validate_result(checked, email, config, url)
 
     elif config.email is not None:
         email = config.email
-        checked = check_email(url, email, config['tor'])
+        checked = check_email(url, email, config['tor'], config['timeout'])
         validate_result(checked, email, config, url)