Feature v2.4.16 新增 --authorization-file 参数

README.md

@@ -3,6 +3,18 @@
 ![Fofa-hack](./images/logo.png)
 
 简体中文 | [English](./docs/EN_README.md)
+
+### 公告
+> 在[issue79](https://github.com/Cl0udG0d/Fofa-hack/issues/79) 和 [issue78](https://github.com/Cl0udG0d/Fofa-hack/issues/78) 基础上的调研发现
+> 
+> fofa对游客的请求进行了限制 , Fofa-hack 程序本身没有问题
+> 
+> 如果你使用的时候发现获取不到数据 , 请使用最新版本中的多authorization实现数据获取
+> 
+> python fofa.py -k index --authorization-file au.txt
+> 
+> 以及... 如果大家有注册的多余没用的账号,也可以把authorization提交到 [issue](https://github.com/Cl0udG0d/Fofa-hack/issues/80)里面来实现共享
+
 ### 简介
 
 PS: 感谢[FOFA](https://fofa.info/)提供这么好的测绘工具
@@ -33,21 +45,19 @@ PS: 感谢[FOFA](https://fofa.info/)提供这么好的测绘工具
 ```shell
 Fofa-hack>python fofa.py -h
 
-             ____  ____  ____  ____
-            | ===|/ () \| ===|/ () \
-            |__|  \____/|__| /__/\__\
-                 _   _   ____   ____  __  __
+             ____  ____  ____  ____      
+            | ===|/ () \| ===|/ () \     
+            |__|  \____/|__| /__/\__\    
+                 _   _   ____   ____  __  __ 
                 | |_| | / () \ / (__`|  |/  /
-                |_| |_|/__/\__\\____)|__|\__\ V2.4.15
+                |_| |_|/__/\__\\____)|__|\__\ V2.4.16
 
                 公众号: 黑糖安全
+            
+usage: fofa.py [-h] (--keyword KEYWORD | --inputfile INPUTFILE | --base BASE | --iconurl ICONURL | --iconfile ICONFILE) [--timesleep TIMESLEEP] [--timeout TIMEOUT] [--endcount ENDCOUNT] [--level LEVEL] [--output OUTPUT]
+               [--outputname OUTPUTNAME] [--fuzz] [--proxy-type {socks4,socks5,http}] [--authorization AUTHORIZATION] [--authorization-file AUTHORIZATION_FILE] [--proxy PROXY | --proxy-url PROXY_URL | --proxy-file PROXY_FILE]
 
-usage: fofa.py [-h] (--keyword KEYWORD | --inputfile INPUTFILE | --base BASE | --iconurl ICONURL | --iconfile ICONFILE)
-               [--timesleep TIMESLEEP] [--timeout TIMEOUT] [--endcount ENDCOUNT] [--level LEVEL] [--output OUTPUT]
-               [--outputname OUTPUTNAME] [--fuzz] [--proxy-type {socks4,socks5,http}] [--authorization AUTHORIZATION]
-               [--proxy PROXY | --proxy-url PROXY_URL | --proxy-file PROXY_FILE]
-
-Fofa-hack v2.4.15 使用说明
+Fofa-hack v2.4.16 使用说明
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -75,11 +85,14 @@ optional arguments:
                         代理类型,默认为http
   --authorization AUTHORIZATION
                         指定Authorization值
+  --authorization-file AUTHORIZATION_FILE
+                        从文件中读取authorization列表 --authorization-file authorization.txt
   --proxy PROXY         指定代理,代理格式 --proxy '127.0.0.1:7890'
   --proxy-url PROXY_URL
                         指定代理url，即访问URL响应为proxy,代理格式 --proxy-url http://127.0.0.1/proxy_pool/get
   --proxy-file PROXY_FILE
                         指定txt格式的代理文件,按行分割,代理格式 --proxy-file proxy.txt
+
 ```
 
 爬取的去重结果会存储到`final_fofaHack.txt`文件中

core/fofaMain.py

@@ -288,6 +288,33 @@ def resetCityKeyword(self, keyURL, key):
                 return city
         return None
 
+    def check_authorization_is_available(self):
+
+        # au = config.AUTHORIZATION_LIST.pop()
+        # print(au)
+        # print(config.AUTHORIZATION_LIST)
+        while len(config.AUTHORIZATION_LIST)>0:
+            config.AUTHORIZATION = config.AUTHORIZATION_LIST.pop()
+
+            try:
+                request_profile_url = "https://api.fofa.info/v1/m/profile"
+                rep = requests.get(request_profile_url, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=10)
+                limit_num = json.loads(rep.text)["data"]["info"]["data_limit"]["web_data"]
+
+                request_month_url = "https://api.fofa.info/v1/m/data_usage/month"
+                rep = requests.get(request_month_url, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=10)
+                available_num = json.loads(rep.text)["data"]["web_data"]
+
+                if available_num + 50 < limit_num:
+                    config.AUTHORIZATION_LIST.append(config.AUTHORIZATION)
+                    return True
+
+            except Exception as e:
+                print(
+                    "\033[1;31m[-] error:AUTHORIZATION测试错误 {}\033[0m".format(e))
+                pass
+        return False
+
     def setIndexTimestamp(self, searchbs64, timestamp_index):
         """
         设置时间列表
@@ -296,8 +323,14 @@ def setIndexTimestamp(self, searchbs64, timestamp_index):
         @return:
         """
         try:
+            if config.AUTHORIZATION_FILE:
+                if not self.check_authorization_is_available():
+                    print("\033[1;31m[-] error:{}\033[0m".format(
+                        "authorization获取数据均达本月上限或authorization存在错误"))
+                    exit(0)
+
             request_url = getUrl(searchbs64)
-            # print(request_url)
+
             rep = requests.get(request_url, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=self.timeout,
                                proxies=self.get_proxy())
             # print(rep.text)

docs/CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG 代码变更记录
 
+### 2.4.16
++ 新增 --authorization-file 参数
+
 ### 2.4.15
 + 添加批量从文件使用代理的功能 --proxy-file
 + 添加使用网址代理的功能 --proxy-url

fofa.py

@@ -56,6 +56,7 @@ def main():
 
     parser.add_argument('--proxy-type',choices=['socks4','socks5', 'http'], help=_("代理类型,默认为http"),default='http')
     parser.add_argument('--authorization', type=str, help="指定Authorization值")
+    parser.add_argument('--authorization-file', type=str, help="从文件中读取authorization列表 --authorization-file authorization.txt")
 
     proxy_group = parser.add_mutually_exclusive_group()
     proxy_group.add_argument('--proxy', help=_("指定代理,代理格式 --proxy '127.0.0.1:7890'"))
@@ -65,6 +66,12 @@ def main():
     #                     help="运行类型,默认为普通方式")
     args = parser.parse_args()
 
+    if args.authorization_file:
+        config.AUTHORIZATION_FILE = args.authorization_file
+        with open(config.AUTHORIZATION_FILE, 'r') as f:
+            for line in f.readlines():
+                config.AUTHORIZATION_LIST.append(line)
+
     time_sleep = int(args.timesleep)
     timeout = int(args.timeout)
     if args.keyword:

test.py

@@ -1,49 +1,23 @@
+import json
 
-from fofa_hack import fofa
-from tookit import fofaUseragent
-
-import requests,sys,mmh3,codecs
-
-def main():
-    result_generator = fofa.api('protocol="socks5" && "Authentication"', endcount=1000)
-    for data in result_generator:
-        for proxy in data:
-            proxies = {'http': "http://{}".format(proxy), "https": "https://{}".format(proxy)}
-            print(fr'[-] test: ' + str(proxies))
-            try:
-                r = requests.get('https://www.taobao.com/help/getip.php', proxies=proxies, timeout=3)
-                if 'ipCallback' in r.text:
-                    print(fr'[*] success: ' + str(proxies))
-            except requests.exceptions.ConnectionError:
-                pass
-            except requests.exceptions.ReadTimeout:
-                pass
-            except KeyboardInterrupt:
-                print('用户退出')
-                exit()
-            except requests.exceptions.InvalidSchema:
-                print('未检测到pysocks')
-                print('pip install -U requests[socks]')
-                print('pip install pysocks')
-                exit()
-
-
-def get_ip():
-    proxies =  {'http': 'socks5://18.178.209.57:5555', 'https': 'socks5://18.178.209.57:5555'}
-    response = requests.get('https://api64.ipify.org?format=json',proxies=proxies).json()
-    return response["ip"]
-
-def get_location():
-    ip_address = get_ip()
-    response = requests.get(f'https://ipapi.co/{ip_address}/json/').json()
-    location_data = {
-        "ip": ip_address,
-        "city": response.get("city"),
-        "region": response.get("region"),
-        "country": response.get("country_name")
-    }
-    return location_data
-
-
-if __name__ == '__main__':
-    main()
+import requests
+
+from tookit import config, fofaUseragent
+
+
+request_url="https://api.fofa.info/v1/m/profile"
+# au = config.AUTHORIZATION_LIST[0]
+# print(au)
+
+
+rep = requests.get(request_url, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=10)
+print(json.loads(rep.text)["data"]["info"]["data_limit"])
+
+
+request_url2 = "https://api.fofa.info/v1/m/data_usage/month"
+rep = requests.get(request_url2, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=10)
+print(json.loads(rep.text)["data"])
+
+# au = config.AUTHORIZATION_LIST.pop()
+# print(au)
+# print(config.AUTHORIZATION_LIST)

tookit/config.py

@@ -5,9 +5,13 @@
 # @File    : config.py
 # @Github: https://github.com/Cl0udG0d
 
-VERSION_NUM = "2.4.15"
+VERSION_NUM = "2.4.16"
 ROOT_PATH=""
 AUTHORIZATION = ""
+AUTHORIZATION_LIST = [
+
+]
+AUTHORIZATION_FILE =""
 
 ### ============================================================================
 ### 代理相关的配置参数