Skip to content

Commit

Permalink
Implement upload API (#1085)
Browse files Browse the repository at this point in the history 
* set up upload framework

* initial implementation

* add title and summary

* include category id in upload API

* complete title summary args for handling_incoming_file

* clean and refactor

* rename upload to create

* add exception to handle missing payload

* handle missing file better

* implement sha1 verification

* reflect mojo upload object

* add thumbnail generation for upload api

* remove utf downgrade

* standardize api response

* put upload API behind login

* fix error

* add status for 400 messages, add sc

* misplaced status code

* edit documentation (#12)

* edit documentation

* temporarily change file and data to path

* move path to query (should be data/file)

* remove upload file parameters

* implement redis lock

* move checksum mismatch to 417 and add utf downgrade handler

* show error reason for file upload

* handle_incoming_file return http status code

* be more explicit with responses

* Update lib/LANraragi/Controller/Api/Archive.pm

Co-authored-by: Difegue <8237712+Difegue@users.noreply.github.com>

* edit response when fail move file

* no need for success status

* upload api docs

* this is why i have a chatgpt subscription

* add redis lock response code

* Update tools/Documentation/api-documentation/archive-api.md

Co-authored-by: Difegue <8237712+Difegue@users.noreply.github.com>

* Update tools/Documentation/api-documentation/archive-api.md

Co-authored-by: Difegue <8237712+Difegue@users.noreply.github.com>

* Update tools/Documentation/api-documentation/archive-api.md

Co-authored-by: Difegue <8237712+Difegue@users.noreply.github.com>

* replace utf downgrade with redis encode

---------

Co-authored-by: Difegue <8237712+Difegue@users.noreply.github.com>
  • Loading branch information
@psilabs-dev @Difegue
psilabs-dev and Difegue authored Nov 19, 2024
1 parent 6f89fcc commit 53dc4d8
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 14 deletions.
163 changes: 162 additions & 1 deletion lib/LANraragi/Controller/Api/Archive.pm
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
package LANraragi::Controller::Api::Archive;
use Mojo::Base 'Mojolicious::Controller';

use Digest::SHA qw(sha1_hex);
use Redis;
use Encode;
use Storable;
use Mojo::JSON qw(decode_json);
use Scalar::Util qw(looks_like_number);

use LANraragi::Utils::Generic qw(render_api_response);
use File::Temp qw(tempdir);
use File::Basename;
use File::Find;

use LANraragi::Utils::Archive qw(extract_thumbnail);
use LANraragi::Utils::Generic qw(render_api_response is_archive get_bytelength);
use LANraragi::Utils::Database qw(get_archive_json set_isnew);
use LANraragi::Utils::Logging qw(get_logger);

use LANraragi::Model::Archive;
use LANraragi::Model::Category;
Expand Down Expand Up @@ -107,6 +114,160 @@ sub serve_file {
$self->render_file( filepath => $file );
}

# Create a file archive along with any metadata.
# adapted from Upload.pm
sub create_archive {
my $self = shift;

my $logger = get_logger( "Archive API ", "lanraragi");
my $redis = LANraragi::Model::Config->get_redis;

# receive uploaded file
my $upload = $self->req->upload('file');
my $expected_checksum = $self->req->param('file_checksum'); # optional

# require file
if ( ! defined $upload || !$upload ) {
return $self->render(
json => {
operation => "upload",
success => 0,
error => "No file attached"
},
status => 400
);
}

# checksum verification stage.
if ( $expected_checksum ) {
my $file_content = $upload->slurp;
my $actual_checksum = sha1_hex($file_content);
if ( $expected_checksum ne $actual_checksum ) {
return $self->render(
json => {
operation => "upload",
success => 0,
error => "Checksum mismatch: expected $expected_checksum, got $actual_checksum."
},
status => 417
);
}
}

my $filename = $upload->filename;
my $uploadMime = $upload->headers->content_type;
$filename = LANraragi::Utils::Database::redis_encode( $filename );

# lock resource
my $lock = $redis->setnx( "upload:$filename", 1 );
if ( !$lock ) {
return $self->render(
json => {
operation => "upload",
success => 0,
error => "Locked resource: $filename."
},
status => 423
);
}
$redis->expire( "upload:$filename", 10 );

# metadata extraction
my $catid = $self->req->param('category_id');
my $tags = $self->req->param('tags');
my $title = $self->req->param('title');
my $summary = $self->req->param('summary');

# return error if archive is not supported.
if ( !is_archive($filename) ) {
$redis->del("upload:$filename");
$redis->quit();
return $self->render(
json => {
operation => "upload",
success => 0,
error => "Unsupported file extension ($filename)"
},
status => 415
);
}

# Move file to a temp folder (not the default LRR one)
my $tempdir = tempdir();

my ( $fn, $path, $ext ) = fileparse( $filename, qr/\.[^.]*/ );
my $byte_limit = LANraragi::Model::Config->enable_cryptofs ? 143 : 255;

$filename = $fn;
while ( get_bytelength( $filename . $ext . ".upload") > $byte_limit ) {
$filename = substr( $filename, 0, -1 );
}
$filename = $filename . $ext;

my $tempfile = $tempdir . '/' . $filename;
if ( !$upload->move_to($tempfile) ) {
$logger->error("Could not move uploaded file $filename to $tempfile");
$redis->del("upload:$filename");
$redis->quit();
return $self->render(
json => {
operation => "upload",
success => 0,
error => "Couldn't move uploaded file to temporary location."
},
status => 500
);
}

# Update $tempfile to the exact reference created by the host filesystem
# This is done by finding the first (and only) file in $tempdir.
find(
sub {
return if -d $_;
$tempfile = $File::Find::name;
$filename = $_;
},
$tempdir
);

my ( $status_code, $id, $response_title, $message ) = LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tags, $title, $summary );

# post-processing thumbnail generation
my %hash = $redis->hgetall($id);
my ( $thumbhash ) = @hash{qw(thumbhash)};
unless ( length $thumbhash ) {
$logger->info("Thumbnail hash invalid, regenerating.");
my $thumbdir = LANraragi::Model::Config->get_thumbdir;
$thumbhash = "";
extract_thumbnail( $thumbdir, $id, 0, 1 );
$thumbhash = $redis->hget( $id, "thumbhash" );
$thumbhash = LANraragi::Utils::Database::redis_decode($thumbhash);
}
$redis->del("upload:$filename");
$redis->quit();

unless ( $status_code == 200 ) {
return $self->render(
json => {
operation => "upload",
success => 0,
error => $message,
id => $id
},
status => $status_code
);
}

return $self->render(
json => {
operation => "upload",
success => 1,
id => $id
},
status => 200
);
}

# Serve an archive page from the temporary folder, using RenderFile.
sub serve_page {
my $self = shift;
Expand Down
30 changes: 20 additions & 10 deletions lib/LANraragi/Model/Upload.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use File::Temp qw(tempdir);
use File::Find qw(find);
use File::Copy qw(move);

use LANraragi::Utils::Database qw(invalidate_cache compute_id);
use LANraragi::Utils::Logging qw(get_logger);
use LANraragi::Utils::Database qw(invalidate_cache compute_id set_title set_summary);
use LANraragi::Utils::Logging qw(get_logger);
use LANraragi::Utils::Database qw(redis_encode);
use LANraragi::Utils::Generic qw(is_archive get_bytelength);
use LANraragi::Utils::String qw(trim trim_CRLF trim_url);
Expand All @@ -29,18 +29,18 @@ use LANraragi::Model::Category;
# The file will be added to a category, if its ID is specified.
# You can also specify tags to add to the metadata for the processed file before autoplugin is ran. (if it's enabled)
#
# Returns a status value, the ID and title of the file, and a status message.
# Returns an HTTP status code, the ID and title of the file, and a status message.
sub handle_incoming_file {

my ( $tempfile, $catid, $tags ) = @_;
my ( $tempfile, $catid, $tags, $title, $summary ) = @_;
my ( $filename, $dirs, $suffix ) = fileparse( $tempfile, qr/\.[^.]*/ );
$filename = $filename . $suffix;
my $logger = get_logger( "File Upload/Download", "lanraragi" );

# Check if file is an archive
unless ( is_archive($filename) ) {
$logger->debug("$filename is not an archive, halting upload process.");
return ( 0, "deadbeef", $filename, "Unsupported File Extension ($filename)" );
return ( 415, "deadbeef", $filename, "Unsupported File Extension ($filename)" );
}

# Compute an ID here
Expand Down Expand Up @@ -71,7 +71,7 @@ sub handle_incoming_file {
? "This file already exists in the Library." . $suffix
: "A file with the same name is present in the Library." . $suffix;

return ( 0, $id, $filename, $msg );
return ( 409, $id, $filename, $msg );
}

# If we are replacing an existing one, just remove the old one first.
Expand Down Expand Up @@ -109,19 +109,29 @@ sub handle_incoming_file {
}
}

# Set title
if ($title) {
set_title( $id, $title );
}

# Set summary
if ($summary) {
set_summary( $id, $summary );
}

# Move the file to the content folder.
# Move to a .upload first in case copy to the content folder takes a while...
move( $tempfile, $output_file . ".upload" )
or return ( 0, $id, $name, "The file couldn't be moved to your content folder: $!" );
or return ( 500, $id, $name, "The file couldn't be moved to your content folder: $!" );

# Then rename inside the content folder itself to proc Shinobu.
move( $output_file . ".upload", $output_file )
or return ( 0, $id, $name, "The file couldn't be renamed in your content folder: $!" );
or return ( 500, $id, $name, "The file couldn't be renamed in your content folder: $!" );

# If the move didn't signal an error, but still doesn't exist, something is quite spooky indeed!
# Really funky permissions that prevents viewing folder contents?
unless ( -e $output_file ) {
return ( 0, $id, $name, "The file couldn't be moved to your content folder!" );
return ( 500, $id, $name, "The file couldn't be moved to your content folder!" );
}

# Now that the file has been copied, we can add the timestamp tag and calculate pagecount.
Expand Down Expand Up @@ -157,7 +167,7 @@ sub handle_incoming_file {
# Invalidate search cache ourselves, Shinobu won't do it since the file is already in the database
invalidate_cache();

return ( 1, $id, $name, $successmsg );
return ( 200, $id, $name, $successmsg );
}

# Download the given URL, using the given Mojo::UserAgent object.
Expand Down
7 changes: 4 additions & 3 deletions lib/LANraragi/Utils/Minion.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,8 @@ sub add_tasks {
$logger->info("Processing uploaded file $file...");

# Since we already have a file, this goes straight to handle_incoming_file.
my ( $status, $id, $title, $message ) = LANraragi::Model::Upload::handle_incoming_file( $file, $catid, "" );

my ( $status_code, $id, $title, $message ) = LANraragi::Model::Upload::handle_incoming_file( $file, $catid, "", "", "" );
my $status = $status_code == 200 ? 1 : 0;
$job->finish(
{ success => $status,
id => $id,
Expand Down Expand Up @@ -253,7 +253,8 @@ sub add_tasks {
my $tag = "source:$og_url";

# Hand off the result to handle_incoming_file
my ( $status, $id, $title, $message ) = LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tag );
my ( $status_code, $id, $title, $message ) = LANraragi::Model::Upload::handle_incoming_file( $tempfile, $catid, $tag, "", "" );
my $status = $status_code == 200 ? 1 : 0;

$job->finish(
{ success => $status,
Expand Down
1 change: 1 addition & 0 deletions lib/LANraragi/Utils/Routing.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ sub apply_routes {
$public_api->get('/api/archives/:id/categories')->to('api-archive#get_categories');
$public_api->get('/api/archives/:id/tankoubons')->to('api-tankoubon#get_tankoubons_file');
$public_api->get('/api/archives/:id/metadata')->to('api-archive#serve_metadata');
$logged_in_api->put('/api/archives/upload')->to('api-archive#create_archive');
$logged_in_api->put('/api/archives/:id/thumbnail')->to('api-archive#update_thumbnail');
$logged_in_api->put('/api/archives/:id/metadata')->to('api-archive#update_metadata');
$logged_in_api->delete('/api/archives/:id')->to('api-archive#delete_archive');
Expand Down
Loading

0 comments on commit 53dc4d8

Please sign in to comment.