Skip to content

Commit

Permalink
fix some lint
Browse files Browse the repository at this point in the history
  • Loading branch information
noisecode3 committed Nov 11, 2024
1 parent e958cc8 commit 56797e6
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 46 deletions.
2 changes: 2 additions & 0 deletions database/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[flake8]
max-line-length = 100
49 changes: 22 additions & 27 deletions database/https.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,16 @@
import get_leaf_cert
import data_factory


class AcquireLock:
"""
Create a TCP socket to ensure a single instance.
This class creates a TCP socket that binds to a specific port. If an instance
of this class is already running, it will log an error and exit.
This class creates a TCP socket that binds to a specific port. If an instance
of this class is already running, it will log an error and exit.
Example usage:
```python
lock = AcquireLock()
try:
Expand Down Expand Up @@ -65,7 +66,6 @@ def __init__(self):
self.misconfigured_server = False
self.leaf_cert = None


def validate_url(self, url):
"""Limit to used domains."""
allowed_domains = (
Expand All @@ -80,7 +80,6 @@ def validate_url(self, url):
logging.error("Invalid URL domain: %s", url)
sys.exit(1)


def head(self, curl):
"""Take the curl object to head state, with redirect."""
if isinstance(curl, pycurl.Curl):
Expand Down Expand Up @@ -108,7 +107,6 @@ def head(self, curl):
return buffer.getvalue().decode('utf-8')
return ""


def validate_data_type(self, content_type):
"""Limit to used data types."""
valid_content_types = {
Expand All @@ -118,14 +116,13 @@ def validate_data_type(self, content_type):
'image/jpeg',
'image/png',
'text/html',
'head' # this is not MIME
'head' # this is not MIME
}

if content_type not in valid_content_types:
logging.error("Invalid content type: %s", content_type)
sys.exit(1)


def set_leaf(self, curl):
"""Write the certificate to a temporary file manually"""
if self.leaf_cert is None:
Expand All @@ -141,7 +138,6 @@ def set_leaf(self, curl):
curl.setopt(pycurl.CAINFO, temp_cert_path)
return temp_cert_path


def get_leaf(self, url):
"""Check if we need to grab the first certificate"""
if not self.misconfigured_server:
Expand All @@ -164,7 +160,6 @@ def setup_before_get_response(self, url, content_type):
if url.startswith("https://www.trle.net/") and not self.misconfigured_server:
self.get_leaf(url)


def get_response(self, url, content_type):
"""Handle all https requests"""
self.setup_before_get_response(url, content_type)
Expand Down Expand Up @@ -217,7 +212,7 @@ def get_response(self, url, content_type):
break

except pycurl.error as curl_error:
#if curl_error.args[0] == 60: # SSL certificate error
# if curl_error.args[0] == 60: # SSL certificate error
logging.error("Request failed: %s", curl_error)
retries += 1
if retries >= max_retries:
Expand All @@ -230,7 +225,6 @@ def get_response(self, url, content_type):

return self.close_response(curl, headers, response_buffer, content_type)


def close_response(self, curl, headers, response_buffer, content_type):
"""Pack response and close curl"""
if curl is None:
Expand All @@ -251,11 +245,10 @@ def close_response(self, curl, headers, response_buffer, content_type):
response = self.pack_response_buffer(content_type, response_buffer)
curl.close()
return response
logging.error("Unexpected content type: %s, expected %s", \
response_content_type, content_type)
logging.error("Unexpected content type: %s, expected %s",
response_content_type, content_type)
sys.exit(1)


def pack_response_buffer(self, content_type, response_buffer):
"""Validate and return the response based on content type"""
if content_type == 'text/html':
Expand All @@ -276,7 +269,6 @@ def pack_response_buffer(self, content_type, response_buffer):
logging.error("Unsupported content type: %s", content_type)
return None


def extract_content_type(self, headers):
"""Read the header lines to look for content-type"""

Expand Down Expand Up @@ -313,11 +305,12 @@ def progress_callback(self, total_to_download, downloaded, total_to_upload, uplo
if total_to_download > 0:
if self.progress_bar is None:
# Initialize the progress bar if it's not set
self.progress_bar= tqdm(total=total_to_download,
unit='B',
unit_scale=True,
unit_divisor=1024,
desc="Downloading")
self.progress_bar = tqdm(total=total_to_download,
unit='B',
unit_scale=True,
unit_divisor=1024,
desc="Downloading")

self.progress_bar.update(downloaded - self.progress_bar.n) # Update the progress bar
self.progress_bar.total = total_to_download
return 0 # Returning 0 means to continue
Expand Down Expand Up @@ -389,11 +382,12 @@ def download_file(self, url):
curl.setopt(pycurl.WRITEDATA, self.buffer)

# Enable progress meter
self.progress_bar= tqdm(total=total_size,
unit='B',
unit_scale=True,
unit_divisor=1024,
desc="Downloading")
self.progress_bar = tqdm(total=total_size,
unit='B',
unit_scale=True,
unit_divisor=1024,
desc="Downloading")

curl.setopt(pycurl.NOPROGRESS, False)
curl.setopt(pycurl.XFERINFOFUNCTION, self.progress_callback)

Expand Down Expand Up @@ -435,6 +429,7 @@ def download_file(self, url):
REQUEST_HANDLER = RequestHandler()
DOWNLOADER = Downloader()


def get(url, content_type):
"""
Get server response from TRLE or Trcustom hosts
Expand Down
13 changes: 7 additions & 6 deletions database/scrape.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,17 @@
# Some basic URL converters
###############################################################################


def trle_search_parser(url):
"""
Prepares a URL for level title searches on TRLE by encoding special characters.
Note: This function should generally be avoided in favor of searching the local
Note: This function should generally be avoided in favor of searching the local
database, as it may not fully cover all cases or include recent updates.
Args:
url (str): The original URL string to be formatted for TRLE search compatibility.
Returns:
str: The formatted URL with specific characters
replaced to match TRLE encoding requirements.
Expand All @@ -46,10 +47,10 @@ def trle_search_parser(url):
def url_postfix(url):
"""
Extracts the file extension from a URL without the leading dot.
Args:
url (str): The URL to extract the file extension from.
Returns:
str: The file extension without the leading dot,
or an empty string if no extension is present.
Expand Down
45 changes: 32 additions & 13 deletions src/Network.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
*/

#include "Network.h"
#include <curl/curl.h>
#include <iostream>
#include <string>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <curl/curl.h>

namespace ssl = boost::asio::ssl;
using tcp = boost::asio::ip::tcp;
Expand All @@ -44,7 +44,7 @@ std::string get_ssl_certificate(const std::string& host) {
BIO* bio = BIO_new(BIO_s_mem());
PEM_write_bio_X509(bio, cert);
char* cert_str = nullptr;
long cert_len = BIO_get_mem_data(bio, &cert_str);
qint64 cert_len = BIO_get_mem_data(bio, &cert_str);
std::string cert_buffer(cert_str, cert_len);

BIO_free(bio);
Expand Down Expand Up @@ -93,7 +93,7 @@ int Downloader::progress_callback(

// Emit signal only if progress has increased by at least 1%
static int lastEmittedProgress = 0;
if ((int)progress == 0) lastEmittedProgress = 0;
if (static_cast<int>(progress) == 0) lastEmittedProgress = 0;
if (static_cast<int>(progress) > lastEmittedProgress)
{
static Downloader& instance = Downloader::getInstance();
Expand Down Expand Up @@ -122,8 +122,16 @@ int Downloader::getStatus()

void Downloader::saveToFile(const QByteArray& data, const QString& filePath)
{
QFileInfo fileInfo(filePath);

if (fileInfo.exists() && !fileInfo.isFile()) {
qDebug() << "Error: The zip path is not a regular file." << filePath;
return;
}

QFile file(filePath);
if (file.open(QIODevice::WriteOnly))

if (file.open(QIODevice::WriteOnly)) // flawfinder: ignore
{
file.write(data);
file.close();
Expand All @@ -144,9 +152,17 @@ void Downloader::run()
QByteArray byteArray = urlString.toUtf8();
const char* url_cstring = byteArray.constData();

QString filePath = levelDir_m.absolutePath() + QDir::separator() + file_m;
const QString filePath = levelDir_m.absolutePath() +
QDir::separator() + file_m;

FILE* file = fopen(filePath.toUtf8().constData(), "wb");
QFileInfo fileInfo(filePath);

if (fileInfo.exists() && !fileInfo.isFile()) {
qDebug() << "Error: The zip path is not a regular file." << filePath;
return;
}

FILE* file = fopen(filePath.toUtf8(), "wb"); // flawfinder: ignore
if (!file)
{
qDebug() << "Error opening file for writing:" << filePath;
Expand All @@ -162,22 +178,25 @@ void Downloader::run()

// Set up the in-memory blob for curl to use
curl_blob blob;
blob.data = const_cast<void*>(static_cast<const void*>(cert_buffer.data()));
//blob.data = cert_buffer.data();
blob.data = cert_buffer.data();
blob.len = cert_buffer.size();
blob.flags = CURL_BLOB_COPY;

curl_easy_setopt(curl, CURLOPT_URL, url_cstring);
curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &blob);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Downloader::write_callback);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
Downloader::write_callback);

curl_easy_setopt(curl, CURLOPT_WRITEDATA, &writeData);

// Follow redirects
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

// Enable progress meter
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, Downloader::progress_callback);
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
Downloader::progress_callback);

curl_easy_setopt(curl, CURLOPT_XFERINFODATA, nullptr);

// Perform the download
Expand All @@ -186,16 +205,16 @@ void Downloader::run()
if (res != CURLE_OK)
{
status_m = 1;
qDebug() << "curl_easy_perform() failed:" << curl_easy_strerror(res);
qDebug() << "CURL failed:" << curl_easy_strerror(res);
// we need to catch any of those that seem inportant here to the GUI
// and reset GUI state
// https://curl.se/libcurl/c/libcurl-errors.html
if(res == 6 || res == 7 || res == 28 || res == 35)
if (res == 6 || res == 7 || res == 28 || res == 35)
{
emit this->networkWorkErrorSignal(1);
QCoreApplication::processEvents();
}
else if(res == CURLE_PEER_FAILED_VERIFICATION)
else if (res == CURLE_PEER_FAILED_VERIFICATION)
{
emit this->networkWorkErrorSignal(2);
QCoreApplication::processEvents();
Expand Down

0 comments on commit 56797e6

Please sign in to comment.