Skip to content

Commit

Permalink
Improve deobfuscation (#469)
Browse files Browse the repository at this point in the history
- improved deobfuscation:
  - enhanced parsing of the subject field in NZB files, along with improved validation of downloaded article filenames 
  - added a check for filename obfuscation after unpacking; excessively obfuscated filenames are renamed to the NZB filename if the NZB filename is not also obfuscated
- new options:
  - "RenameAfterUnpack" - enables/disables renaming of extracted and obfuscated files using the NZB filename
  - "RenameIgnoreExt" - a comma separated list of file extensions to ignore when renaming files after unpacking
- new value for field "Status" in API-method "listgroups": "POST_UNPACK_RENAMING"
  • Loading branch information
dnzbk authored Dec 17, 2024
1 parent 3eb54ff commit 9f755bf
Show file tree
Hide file tree
Showing 23 changed files with 626 additions and 152 deletions.
7 changes: 7 additions & 0 deletions daemon/main/Options.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
*
* Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
* Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
* Copyright (C) 2024 Denis <denis@nzbget.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
Expand Down Expand Up @@ -93,6 +94,8 @@ static const char* OPTION_PARSCAN = "ParScan";
static const char* OPTION_PARQUICK = "ParQuick";
static const char* OPTION_POSTSTRATEGY = "PostStrategy";
static const char* OPTION_FILENAMING = "FileNaming";
static const char* OPTION_RENAMEAFTERUNPACK = "RenameAfterUnpack";
static const char* OPTION_RENAMEIGNOREEXT = "RenameIgnoreExt";
static const char* OPTION_PARRENAME = "ParRename";
static const char* OPTION_PARBUFFER = "ParBuffer";
static const char* OPTION_PARTHREADS = "ParThreads";
Expand Down Expand Up @@ -480,6 +483,8 @@ void Options::InitDefaults()
SetOption(OPTION_PARQUICK, "yes");
SetOption(OPTION_POSTSTRATEGY, "sequential");
SetOption(OPTION_FILENAMING, "article");
SetOption(OPTION_RENAMEAFTERUNPACK, "yes");
SetOption(OPTION_RENAMEIGNOREEXT, ".zip, .7z, .rar, .par2");
SetOption(OPTION_PARRENAME, "yes");
SetOption(OPTION_PARBUFFER, "16");
SetOption(OPTION_PARTHREADS, "0");
Expand Down Expand Up @@ -703,6 +708,7 @@ void Options::InitOptions()
m_parIgnoreExt = GetOption(OPTION_PARIGNOREEXT);
m_unpackIgnoreExt = GetOption(OPTION_UNPACKIGNOREEXT);
m_shellOverride = GetOption(OPTION_SHELLOVERRIDE);
m_renameIgnoreExt = GetOption(OPTION_RENAMEIGNOREEXT);

m_downloadRate = ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024;
m_articleTimeout = ParseIntValue(OPTION_ARTICLETIMEOUT, 10);
Expand Down Expand Up @@ -773,6 +779,7 @@ void Options::InitOptions()
m_urlForce = (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues);
m_certCheck = (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues);
m_reorderFiles = (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues);
m_renameAfterUnpack = (bool)ParseEnumValue(OPTION_RENAMEAFTERUNPACK, BoolCount, BoolNames, BoolValues);

const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" };
const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses };
Expand Down
5 changes: 5 additions & 0 deletions daemon/main/Options.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
*
* Copyright (C) 2004 Sven Henkel <sidddy@users.sourceforge.net>
* Copyright (C) 2007-2019 Andrey Prygunkov <hugbug@users.sourceforge.net>
* Copyright (C) 2024 Denis <denis@nzbget.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
Expand Down Expand Up @@ -294,6 +295,8 @@ class Options
bool GetUnpackPauseQueue() { return m_unpackPauseQueue; }
const char* GetExtCleanupDisk() { return m_extCleanupDisk; }
const char* GetParIgnoreExt() { return m_parIgnoreExt; }
const char* GetRenameIgnoreExt() { return m_renameIgnoreExt; }
bool GetRenameAfterUnpack() { return m_renameAfterUnpack; }
const char* GetUnpackIgnoreExt() { return m_unpackIgnoreExt; }
int GetFeedHistory() { return m_feedHistory; }
bool GetUrlForce() { return m_urlForce; }
Expand Down Expand Up @@ -444,6 +447,8 @@ class Options
int m_dailyQuota = 0;
bool m_reorderFiles = false;
EFileNaming m_fileNaming = nfArticle;
bool m_renameAfterUnpack = true;
CString m_renameIgnoreExt;
int m_downloadRate = 0;

// Application mode
Expand Down
144 changes: 144 additions & 0 deletions daemon/postprocess/PostUnpackRenamer.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/*
* This file is part of nzbget. See <https://nzbget.com>.
*
* Copyright (C) 2024 Denis <denis@nzbget.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

#include "nzbget.h"

#include "PostUnpackRenamer.h"
#include "FileSystem.h"
#include "Deobfuscation.h"

namespace PostUnpackRenamer
{
void Controller::StartJob(PostInfo* postInfo)
{
Controller* controller = new (std::nothrow) Controller();

if (!controller)
{
error("Failed to allocate memory for PostUnpackRenamer::Controller");
return;
}

controller->m_postInfo = postInfo;
controller->SetAutoDestroy(false);

postInfo->SetPostThread(controller);

controller->Start();
}

void Controller::Run()
{
{
GuardedDownloadQueue guard = DownloadQueue::Guard();

m_name = m_postInfo->GetNzbInfo()->GetName();
m_dstDir = m_postInfo->GetNzbInfo()->GetDestDir();
}

std::string infoName = "Post-unpack renaming for " + m_name;
SetInfoName(infoName.c_str());

if (Deobfuscation::IsExcessivelyObfuscated(m_name))
{
PrintMessage(Message::mkWarning,
"Skipping Post-unpack renaming. NZB filename %s is excessively obfuscated which makes renaming unreliable.",
m_name.c_str()
);
m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
NzbInfo::PostUnpackRenamingStatus::Skipped
);
m_postInfo->SetWorking(false);
return;
}

bool ok = RenameFiles(m_dstDir, m_name);

GuardedDownloadQueue guard = DownloadQueue::Guard();
if (ok)
{
PrintMessage(Message::mkInfo, "%s successful", infoName.c_str());
m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
NzbInfo::PostUnpackRenamingStatus::Success
);
}
else
{
PrintMessage(Message::mkError, "%s failed", infoName.c_str());
m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus(
NzbInfo::PostUnpackRenamingStatus::Failure
);
}

m_postInfo->SetWorking(false);
}

bool Controller::RenameFiles(const std::string& dir, const std::string& newName)
{
DirBrowser dirBrowser(dir.c_str());
while (const char* fileOrDir = dirBrowser.Next())
{
std::string srcFileOrDir = dir + PATH_SEPARATOR + fileOrDir;

if (FileSystem::DirectoryExists(srcFileOrDir.c_str()))
{
RenameFiles(srcFileOrDir, newName);
continue;
}

if (!Deobfuscation::IsExcessivelyObfuscated(fileOrDir))
{
PrintMessage(Message::mkInfo,
"Filename %s is not excessively obfuscated, no renaming needed.",
fileOrDir
);
continue;
}

std::string dstFile = dir + PATH_SEPARATOR + newName;
dstFile += FileSystem::GetFileExtension(srcFileOrDir).value_or("");

if (Util::MatchFileExt(dstFile.c_str(), g_Options->GetRenameIgnoreExt(), ","))
{
continue;
}

if (FileSystem::MoveFile(srcFileOrDir.c_str(), dstFile.c_str()))
{
PrintMessage(Message::mkInfo, "%s renamed to %s", srcFileOrDir.c_str(), dstFile.c_str());
}
else
{
PrintMessage(Message::mkError,
"Could not rename file %s to %s: %s",
srcFileOrDir.c_str(),
dstFile.c_str(),
*FileSystem::GetLastErrorMessage()
);
}
}

return true;
}

void Controller::AddMessage(Message::EKind kind, const char* text)
{
m_postInfo->GetNzbInfo()->AddMessage(kind, text);
}
}
48 changes: 48 additions & 0 deletions daemon/postprocess/PostUnpackRenamer.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* This file is part of nzbget. See <https://nzbget.com>.
*
* Copyright (C) 2024 Denis <denis@nzbget.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/


#ifndef POST_UNPACK_H
#define POST_UNPACK_H

#include <string>
#include "Thread.h"
#include "ScriptController.h"
#include "DownloadInfo.h"

namespace PostUnpackRenamer
{
class Controller final : public Thread, public ScriptController
{
public:
void Run() override;
static void StartJob(PostInfo* postInfo);

protected:
void AddMessage(Message::EKind kind, const char* text) override;

private:
PostInfo* m_postInfo;
std::string m_name;
std::string m_dstDir;
bool RenameFiles(const std::string& dir, const std::string& nameToRename);
};
}

#endif
17 changes: 17 additions & 0 deletions daemon/postprocess/PrePostProcessor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include "QueueScript.h"
#include "ParParser.h"
#include "DirectUnpack.h"
#include "PostUnpackRenamer.h"

PrePostProcessor::PrePostProcessor()
{
Expand Down Expand Up @@ -844,6 +845,17 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
unpack = false;
}

bool postUnpackRenaming = g_Options->GetRenameAfterUnpack() &&
nzbInfo->GetPostUnpackRenamingStatus() == NzbInfo::PostUnpackRenamingStatus::None &&
nzbInfo->GetDestDir() &&
nzbInfo->GetName() &&
nzbInfo->GetUnpackStatus() != NzbInfo::usFailure &&
nzbInfo->GetUnpackStatus() != NzbInfo::usSpace &&
nzbInfo->GetUnpackStatus() != NzbInfo::usPassword &&
nzbInfo->GetParStatus() != NzbInfo::psFailure &&
nzbInfo->GetParStatus() != NzbInfo::psManual &&
nzbInfo->GetMoveStatus() == NzbInfo::msSuccess;

if (unpack)
{
EnterStage(downloadQueue, postInfo, PostInfo::ptUnpacking);
Expand All @@ -859,6 +871,11 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo
EnterStage(downloadQueue, postInfo, PostInfo::ptMoving);
MoveController::StartJob(postInfo);
}
else if (postUnpackRenaming)
{
EnterStage(downloadQueue, postInfo, PostInfo::ptPostUnpackRenaming);
PostUnpackRenamer::Controller::StartJob(postInfo);
}
else
{
EnterStage(downloadQueue, postInfo, PostInfo::ptExecutingScript);
Expand Down
Loading

0 comments on commit 9f755bf

Please sign in to comment.