Skip to content

Commit

Permalink
Merge pull request #650 from Guovin/dev
Browse files Browse the repository at this point in the history 
Fix and Feat
  • Loading branch information
@Guovin
Guovin authored Dec 11, 2024
2 parents 9844ea3 + 8cfa235 commit 2d5c9d7
Show file tree
Hide file tree
Showing 10 changed files with 509 additions and 457 deletions.
Binary file added config/cache.pkl
View file
Binary file not shown.
8 changes: 5 additions & 3 deletions config/config.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
open_service = True
open_update = True
open_use_old_result = True
open_use_cache = True
open_request = False
source_file = config/demo.txt
final_file = output/result.txt
open_online_search = False
Expand All @@ -24,16 +26,16 @@ ipv6_num = 5
open_m3u_result = True
url_keywords_blacklist = epg.pw,skype.serv00.net,iptv.yjxfz.com,live-hls-web-ajb.getaj.net,live.goodiptv.club,hc73k3dhwo5gfkt.wcetv.com,stream1.freetv.fun,zw9999.cnstream.top,zsntlqj.xicp.net
open_subscribe = True
subscribe_urls = https://live.zbds.top/tv/iptv6.txt,https://live.zbds.top/tv/iptv4.txt,https://live.fanmingming.com/tv/m3u/ipv6.m3u,https://ghp.ci/https://raw.githubusercontent.com/joevess/IPTV/main/home.m3u8,https://aktv.top/live.txt,http://175.178.251.183:6689/live.txt,https://ghp.ci/https://raw.githubusercontent.com/kimwang1978/collect-tv-txt/main/merged_output.txt,https://m3u.ibert.me/txt/fmml_dv6.txt,https://m3u.ibert.me/txt/o_cn.txt,https://m3u.ibert.me/txt/j_iptv.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_CCTV.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_weishi.txt,http://itv.22m.top/ITVBox/tv/tvonline.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/l.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/kk.txt
subscribe_urls = https://iptv.b2og.com/txt/fmml_ipv6.txt,https://ghp.ci/raw.githubusercontent.com/suxuang/myIPTV/main/ipv6.m3u,https://live.zbds.top/tv/iptv6.txt,https://live.zbds.top/tv/iptv4.txt,https://live.fanmingming.com/tv/m3u/ipv6.m3u,https://ghp.ci/https://raw.githubusercontent.com/joevess/IPTV/main/home.m3u8,https://aktv.top/live.txt,http://175.178.251.183:6689/live.txt,https://ghp.ci/https://raw.githubusercontent.com/kimwang1978/collect-tv-txt/main/merged_output.txt,https://m3u.ibert.me/txt/fmml_dv6.txt,https://m3u.ibert.me/txt/o_cn.txt,https://m3u.ibert.me/txt/j_iptv.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_CCTV.txt,https://ghp.ci/https://raw.githubusercontent.com/xzw832/cmys/main/S_weishi.txt,http://itv.22m.top/ITVBox/tv/tvonline.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/l.txt,https://ghp.ci//https://raw.githubusercontent.com/asdjkl6/tv/tv/.m3u/整套直播源/测试/整套直播源/kk.txt
open_multicast = True
open_multicast_foodie = False
open_multicast_foodie = True
open_multicast_fofa = True
multicast_region_list = 全部
multicast_page_num = 1
open_proxy = False
open_driver = False
open_hotel = True
open_hotel_foodie = False
open_hotel_foodie = True
open_hotel_fofa = True
hotel_region_list = 全部
hotel_page_num = 1
Expand Down
2 changes: 2 additions & 0 deletions docs/config.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
| open_service | True | 开启页面服务，用于控制是否启动结果页面服务；如果使用青龙等平台部署，有专门设定的定时任务，需要更新完成后停止运行，可以关闭该功能 |
| open_update | True | 开启更新，用于控制是否更新接口，若关闭则所有工作模式（获取接口和测速）均停止 |
| open_use_old_result | True | 开启使用历史更新结果（包含模板与结果文件的接口），合并至本次更新中 |
| open_use_cache | True | 开启使用本地缓存数据，适用于查询请求失败场景 |
| open_request | False | 开启查询请求，数据来源于网络 |
| open_driver | True | 开启浏览器运行，若更新无数据可开启此模式，较消耗性能 |
| open_proxy | False | 开启代理，自动获取免费可用代理，若更新无数据可开启此模式 |
| source_file | config/demo.txt | 模板文件路径 |
Expand Down
2 changes: 2 additions & 0 deletions docs/config_en.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
| open_service | True | Enable page service, used to control whether to start the result page service; if deployed on platforms like Qinglong with dedicated scheduled tasks, the function can be turned off after updates are completed and the task is stopped |
| open_update | True | Enable updates, if disabled then only the result page service is run |
| open_use_old_result | True | Enable the use of historical update results (including the interface for template and result files) and merge them into the current update |
| open_use_cache | True | Enable the use of local cache data, applicable to the query request failure scenario |
| open_request | False | Enable query request, the data is obtained from the network |
| open_driver | True | Enable browser execution, If there are no updates, this mode can be enabled, which consumes more performance |
| open_proxy | False | Enable proxy, automatically obtains free available proxies, If there are no updates, this mode can be enabled |
| source_file | config/demo.txt | Template file path |
Expand Down
213 changes: 107 additions & 106 deletions updates/fofa/request.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,125 +71,126 @@ async def get_channels_by_fofa(urls=None, multicast=False, callback=None):
"""
Get the channel by FOFA
"""
fofa_urls = urls if urls else get_fofa_urls_from_region_list()
fofa_urls_len = len(fofa_urls)
pbar = tqdm_asyncio(
total=fofa_urls_len,
desc=f"Processing fofa for {'multicast' if multicast else 'hotel'}",
)
start_time = time()
fofa_results = {}
mode_name = "组播" if multicast else "酒店"
if callback:
callback(
f"正在获取Fofa{mode_name}源, 共{fofa_urls_len}个查询地址",
0,
if config.open_use_cache:
fofa_results = get_fofa_region_result_tmp(multicast=multicast)
if config.open_request:
fofa_urls = urls if urls else get_fofa_urls_from_region_list()
fofa_urls_len = len(fofa_urls)
pbar = tqdm_asyncio(
total=fofa_urls_len,
desc=f"Processing fofa for {'multicast' if multicast else 'hotel'}",
)
proxy = None
open_proxy = config.open_proxy
open_driver = config.open_driver
if open_driver:
from driver.setup import setup_driver
open_sort = config.open_sort
if open_proxy:
test_url = fofa_urls[0][0]
proxy = await get_proxy(test_url, best=True, with_test=True)
cancel_event = threading.Event()
hotel_name = constants.origin_map["hotel"]
start_time = time()
mode_name = "组播" if multicast else "酒店"
if callback:
callback(
f"正在获取Fofa{mode_name}源, 共{fofa_urls_len}个查询地址",
0,
)
proxy = None
open_proxy = config.open_proxy
open_driver = config.open_driver
if open_driver:
from driver.setup import setup_driver
open_sort = config.open_sort
if open_proxy:
test_url = fofa_urls[0][0]
proxy = await get_proxy(test_url, best=True, with_test=True)
cancel_event = threading.Event()
hotel_name = constants.origin_map["hotel"]

def process_fofa_channels(fofa_info):
nonlocal proxy
if cancel_event.is_set():
return {}
fofa_url = fofa_info[0]
results = defaultdict(lambda: defaultdict(list))
driver = None
try:
if open_driver:
driver = setup_driver(proxy)
try:
retry_func(lambda: driver.get(fofa_url), name=fofa_url)
except Exception as e:
if open_proxy:
proxy = get_proxy_next()
def process_fofa_channels(fofa_info):
nonlocal proxy
if cancel_event.is_set():
return {}
fofa_url = fofa_info[0]
results = defaultdict(lambda: defaultdict(list))
driver = None
try:
if open_driver:
driver = setup_driver(proxy)
try:
retry_func(lambda: driver.get(fofa_url), name=fofa_url)
except Exception as e:
if open_proxy:
proxy = get_proxy_next()
driver.close()
driver.quit()
driver = setup_driver(proxy)
driver.get(fofa_url)
page_source = driver.page_source
else:
page_source = retry_func(
lambda: get_source_requests(fofa_url), name=fofa_url
)
if any(keyword in page_source for keyword in ["访问异常", "禁止访问", "资源访问每天限制"]):
cancel_event.set()
raise ValueError("Limited access to fofa page")
fofa_source = re.sub(r"<!--.*?-->", "", page_source, flags=re.DOTALL)
urls = set(re.findall(r"https?://[\w\.-]+:\d+", fofa_source))
if multicast:
region = fofa_info[1]
type = fofa_info[2]
multicast_result = [(url, None, None) for url in urls]
results[region][type] = multicast_result
else:
with ThreadPoolExecutor(max_workers=100) as executor:
futures = [
executor.submit(
process_fofa_json_url,
url,
fofa_info[1],
open_sort,
hotel_name,
)
for url in urls
]
for future in futures:
results = merge_objects(results, future.result())
return results
except ValueError as e:
raise e
except Exception as e:
print(e)
finally:
if driver:
driver.close()
driver.quit()
driver = setup_driver(proxy)
driver.get(fofa_url)
page_source = driver.page_source
else:
page_source = retry_func(
lambda: get_source_requests(fofa_url), name=fofa_url
)
if any(keyword in page_source for keyword in ["访问异常", "禁止访问", "资源访问每天限制"]):
cancel_event.set()
raise ValueError("Limited access to fofa page")
fofa_source = re.sub(r"<!--.*?-->", "", page_source, flags=re.DOTALL)
urls = set(re.findall(r"https?://[\w\.-]+:\d+", fofa_source))
if multicast:
region = fofa_info[1]
type = fofa_info[2]
multicast_result = [(url, None, None) for url in urls]
results[region][type] = multicast_result
else:
with ThreadPoolExecutor(max_workers=100) as executor:
futures = [
executor.submit(
process_fofa_json_url,
url,
fofa_info[1],
open_sort,
hotel_name,
)
for url in urls
]
for future in futures:
results = merge_objects(results, future.result())
return results
except ValueError as e:
raise e
except Exception as e:
print(e)
finally:
if driver:
driver.close()
driver.quit()
pbar.update()
remain = fofa_urls_len - pbar.n
if callback:
callback(
f"正在获取Fofa{mode_name}源, 剩余{remain}个查询地址待获取, 预计剩余时间: {get_pbar_remaining(n=pbar.n, total=pbar.total, start_time=start_time)}",
int((pbar.n / fofa_urls_len) * 100),
)
pbar.update()
remain = fofa_urls_len - pbar.n
if callback:
callback(
f"正在获取Fofa{mode_name}源, 剩余{remain}个查询地址待获取, 预计剩余时间: {get_pbar_remaining(n=pbar.n, total=pbar.total, start_time=start_time)}",
int((pbar.n / fofa_urls_len) * 100),
)

max_workers = 3 if open_driver else 10
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(process_fofa_channels, fofa_url) for fofa_url in fofa_urls
]
try:
for future in as_completed(futures):
result = future.result()
if result:
fofa_results = merge_objects(fofa_results, result)
except ValueError as e:
if "Limited access to fofa page" in str(e):
for future in futures:
future.cancel()
if fofa_results:
update_fofa_region_result_tmp(fofa_results, multicast=multicast)
else:
fofa_results = get_fofa_region_result_tmp(multicast=multicast)
max_workers = 3 if open_driver else 10
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(process_fofa_channels, fofa_url) for fofa_url in fofa_urls
]
try:
for future in as_completed(futures):
result = future.result()
if result:
fofa_results = merge_objects(fofa_results, result)
except ValueError as e:
if "Limited access to fofa page" in str(e):
for future in futures:
future.cancel()
if fofa_results:
update_fofa_region_result_tmp(fofa_results, multicast=multicast)
pbar.n = fofa_urls_len
pbar.update(0)
if callback:
callback(
f"正在获取Fofa{mode_name}源",
100,
)
if not open_driver:
close_session()
pbar.close()
if not open_driver:
close_session()
pbar.close()
return fofa_results


Expand Down
Loading

0 comments on commit 2d5c9d7

Please sign in to comment.