Overhaul the import process to use backgrounds jobs (#287 #843)

Kovah · Oct 6, 2024 · ac0090e · ac0090e
1 parent 1cf9713
commit ac0090e
Show file tree

Hide file tree

Showing 16 changed files with 467 additions and 174 deletions.
diff --git a/app/Actions/ImportHtmlBookmarks.php b/app/Actions/ImportHtmlBookmarks.php
@@ -3,17 +3,15 @@
 namespace App\Actions;
 
 use App\Enums\ModelAttribute;
-use App\Helper\HtmlMeta;
-use App\Helper\LinkIconMapper;
+use App\Jobs\ImportLinkJob;
 use App\Models\Link;
 use App\Models\Tag;
-use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\Log;
 use Shaarli\NetscapeBookmarkParser\NetscapeBookmarkParser;
 
 class ImportHtmlBookmarks
 {
-    protected int $imported = 0;
+    protected int $queued = 0;
     protected int $skipped = 0;
     protected ?Tag $importTag = null;
 
@@ -34,63 +32,23 @@ public function run(string $data, string $userId, bool $generateMeta = true): bo
             'visibility' => ModelAttribute::VISIBILITY_PRIVATE,
         ]);
 
-        foreach ($links as $link) {
+        foreach ($links as $i => $link) {
             if (Link::whereUrl($link['url'])->first()) {
                 $this->skipped++;
                 continue;
             }
 
-            if ($generateMeta) {
-                $linkMeta = (new HtmlMeta)->getFromUrl($link['url']);
-                $title = $link['name'] ?: $linkMeta['title'];
-                $description = $link['description'] ?: $linkMeta['description'];
-            } else {
-                $title = $link['name'];
-                $description = $link['description'];
-            }
-
-            if (isset($link['public'])) {
-                $visibility = $link['public'] ? ModelAttribute::VISIBILITY_PUBLIC : ModelAttribute::VISIBILITY_PRIVATE;
-            } else {
-                $visibility = usersettings('links_default_visibility');
-            }
-            $newLink = new Link([
-                'user_id' => $userId,
-                'url' => $link['url'],
-                'title' => $title,
-                'description' => $description,
-                'icon' => LinkIconMapper::getIconForUrl($link['url']),
-                'visibility' => $visibility,
-            ]);
-            $newLink->created_at = $link['dateCreated']
-                ? Carbon::createFromTimestamp($link['dateCreated'])
-                : Carbon::now();
-            $newLink->updated_at = Carbon::now();
-            $newLink->timestamps = false;
-            $newLink->save();
-
-            $newTags = [$this->importTag->id];
-            if (!empty($link['tags'])) {
-                foreach ($link['tags'] as $tag) {
-                    $newTag = Tag::firstOrCreate([
-                        'user_id' => $userId,
-                        'name' => $tag,
-                        'visibility' => usersettings('tags_default_visibility'),
-                    ]);
-                    $newTags[] = $newTag->id;
-                }
-            }
-            $newLink->tags()->sync($newTags);
+            dispatch(new ImportLinkJob($userId, $link, $this->importTag, $generateMeta))->delay($i * 10);
 
-            $this->imported++;
+            $this->queued++;
         }
 
         return true;
     }
 
-    public function getImportCount(): int
+    public function getQueuedCount(): int
     {
-        return $this->imported;
+        return $this->queued;
     }
 
     public function getSkippedCount(): int

diff --git a/app/Console/Commands/ImportCommand.php b/app/Console/Commands/ImportCommand.php
@@ -13,18 +13,17 @@ class ImportCommand extends Command
 
     protected $signature = 'links:import
                         {filepath : Bookmarks file to import, use absolute paths if stored outside of LinkAce}
-                        {--skip-meta-generation : Whether the automatic generation of titles should be skipped.}
-                        {--skip-check : Whether the links checking should be skipped afterwards}';
+                        {--skip-meta-generation : Whether the automatic generation of titles should be skipped.}';
 
     protected $description = 'Import links from a bookmarks file which is stored locally in your file system.';
 
     public function handle(): void
     {
-        $lookupMeta = true;
+        $generateMeta = true;
 
         if ($this->option('skip-meta-generation')) {
             $this->info('Skipping automatic meta generation.');
-            $lookupMeta = false;
+            $generateMeta = false;
         }
 
         $this->info('You will be asked to select a user who will be the owner of the imported bookmarks now.');
@@ -40,24 +39,19 @@ public function handle(): void
         }
 
         $importer = new ImportHtmlBookmarks;
-        $result = $importer->run($data, $this->user->id, $lookupMeta);
+        $result = $importer->run($data, $this->user->id, $generateMeta);
 
         if ($result === false) {
             $this->error('Error while importing bookmarks. Please check the application logs.');
             return;
         }
 
-        if ($this->option('skip-check')) {
-            $this->info('Skipping link check.');
-        } elseif (config('mail.host') !== null) {
-            Artisan::queue('links:check');
-        } else {
-            $this->warn('Links are configured to be checked, but email is not configured!');
-        }
-
+        $tag = $importer->getImportTag();
         $this->info(trans('import.import_successfully', [
-            'imported' => $importer->getImportCount(),
+            'queued' => $importer->getQueuedCount(),
             'skipped' => $importer->getSkippedCount(),
+
+            'taglink' => sprintf('<a href="%s">%s</a>', route('tags.show', ['tag' => $tag]), $tag->name),
         ]));
     }
 }
diff --git a/app/Console/Commands/RegisterUserCommand.php b/app/Console/Commands/RegisterUserCommand.php
@@ -3,12 +3,13 @@
 namespace App\Console\Commands;
 
 use App\Actions\Fortify\CreateNewUser;
+use App\Enums\Role;
 use Illuminate\Console\Command;
 use Illuminate\Validation\ValidationException;
 
 class RegisterUserCommand extends Command
 {
-    protected $signature = 'registeruser {name? : Username} {email? : User email address}';
+    protected $signature = 'registeruser {name? : Username} {email? : User email address} {--admin}';
     protected $description = 'Register a new user with a given user name and an email address.';
 
     private ?string $userName;
@@ -25,7 +26,7 @@ public function handle(): void
             $this->askForUserDetails();
 
             try {
-                (new CreateNewUser)->create([
+                $newUser = (new CreateNewUser)->create([
                     'name' => $this->userName,
                     'email' => $this->userEmail,
                     'password' => $this->userPassword,
@@ -41,6 +42,10 @@ public function handle(): void
             }
         } while ($this->validationFailed);
 
+        if ($this->option('admin')) {
+            $newUser->assignRole(Role::ADMIN);
+        }
+
         $this->info('User ' . $this->userName . ' registered.');
     }
 

diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
@@ -15,23 +15,11 @@
 
 class Kernel extends ConsoleKernel
 {
-    protected $commands = [
-        CheckLinksCommand::class,
-        CompleteSetupCommand::class,
-        ImportCommand::class,
-        RegisterUserCommand::class,
-        ResetPasswordCommand::class,
-        UpdateLinkThumbnails::class,
-        ListUsersCommand::class,
-        ViewRecoveryCodesCommand::class,
-    ];
-
     protected function schedule(Schedule $schedule): void
     {
-        $schedule->command('links:check')->hourly();
+        $schedule->command('queue:work --queue=default,import')->withoutOverlapping();
 
-        $schedule->command('queue:work --daemon --once')
-            ->withoutOverlapping();
+        $schedule->command('links:check')->hourly();
 
         if (config('backup.backup.enabled')) {
             $schedule->command('backup:clean')->daily()->at('01:00');

diff --git a/app/Http/Controllers/App/ImportController.php b/app/Http/Controllers/App/ImportController.php
@@ -5,19 +5,38 @@
 use App\Actions\ImportHtmlBookmarks;
 use App\Http\Controllers\Controller;
 use App\Http\Requests\DoImportRequest;
+use App\Jobs\ImportLinkJob;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
 use Illuminate\Contracts\View\View;
 use Illuminate\Http\JsonResponse;
+use Illuminate\Support\Facades\DB;
 
 class ImportController extends Controller
 {
-    public function getImport(): View
+    public function form(): View
     {
         return view('app.import.import', [
             'pageTitle' => trans('import.import'),
         ]);
     }
 
+    public function queue(): View
+    {
+        $jobs = DB::table('jobs')
+            ->where('payload', 'LIKE', '%"displayName":' . json_encode(ImportLinkJob::class) . '%')
+            ->paginate(50);
+
+        $failedJobs = DB::table('failed_jobs')
+            ->where('payload', 'LIKE', '%"displayName":' . json_encode(ImportLinkJob::class) . '%')
+            ->paginate(50);
+
+        return view('app.import.queue', [
+            'pageTitle' => trans('import.import'),
+            'jobs' => $jobs,
+            'failed_jobs' => $failedJobs,
+        ]);
+    }
+
     /**
      * Load the provided HTML bookmarks file and save all parsed results as new
      * links including tags. This method is called via an Ajax call to prevent
@@ -42,12 +61,13 @@ public function doImport(DoImportRequest $request): JsonResponse
         }
 
         $tag = $importer->getImportTag();
+
         return response()->json([
             'success' => true,
             'message' => trans('import.import_successfully', [
-                'imported' => $importer->getImportCount(),
+                'queued' => $importer->getQueuedCount(),
                 'skipped' => $importer->getSkippedCount(),
-                'taglink' => sprintf('<a href="%s">%s</a>', route('tags.show', [$tag]), $tag->name),
+                'taglink' => sprintf('<a href="%s">%s</a>', route('tags.show', ['tag' => $tag]), $tag->name),
             ]),
         ]);
     }

diff --git a/app/Jobs/ImportLinkJob.php b/app/Jobs/ImportLinkJob.php
@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enums\ModelAttribute;
+use App\Helper\HtmlMeta;
+use App\Helper\LinkIconMapper;
+use App\Models\Link;
+use App\Models\Tag;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Carbon;
+
+class ImportLinkJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct(
+        public int $userId,
+        public array $link,
+        public Tag $importTag,
+        public bool $generateMeta
+    ) {
+        $this->onQueue('import');
+    }
+
+    public function handle(): void
+    {
+        if ($this->generateMeta) {
+            $linkMeta = (new HtmlMeta)->getFromUrl($this->link['url']);
+            $title = $this->link['name'] ?: $linkMeta['title'];
+            $description = $this->link['description'] ?: $linkMeta['description'];
+        } else {
+            $title = $this->link['name'];
+            $description = $this->link['description'];
+        }
+
+        if (isset($this->link['public'])) {
+            $visibility = $this->link['public'] ? ModelAttribute::VISIBILITY_PUBLIC : ModelAttribute::VISIBILITY_PRIVATE;
+        } else {
+            $visibility = usersettings('links_default_visibility', $this->userId);
+        }
+
+        $newLink = new Link([
+            'user_id' => $this->userId,
+            'url' => $this->link['url'],
+            'title' => $title,
+            'description' => $description,
+            'icon' => LinkIconMapper::getIconForUrl($this->link['url']),
+            'visibility' => $visibility,
+        ]);
+
+        $newLink->created_at = $this->link['dateCreated']
+            ? Carbon::createFromTimestamp($this->link['dateCreated'])
+            : Carbon::now();
+        $newLink->updated_at = Carbon::now();
+        $newLink->timestamps = false;
+        $newLink->save();
+
+        $newTags = [$this->importTag->id];
+
+        if (!empty($this->link['tags'])) {
+            foreach ($this->link['tags'] as $tag) {
+                $newTag = Tag::firstOrCreate([
+                    'user_id' => $this->userId,
+                    'name' => $tag,
+                    'visibility' => usersettings('tags_default_visibility', $this->userId),
+                ]);
+                $newTags[] = $newTag->id;
+            }
+        }
+
+        $newLink->tags()->sync($newTags);
+    }
+}
diff --git a/database/migrations/2024_10_06_211654_update_failed_jobs_table.php b/database/migrations/2024_10_06_211654_update_failed_jobs_table.php
@@ -0,0 +1,25 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    public function up(): void
+    {
+        Schema::table('failed_jobs', function (Blueprint $table) {
+            $table->string('uuid')->unique()->after('id');
+            $table->longText('payload')->change();
+            $table->longText('exception')->change();
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::table('failed_jobs', function (Blueprint $table) {
+            $table->dropColumn(['uuid']);
+            $table->text('payload')->change();
+            $table->text('exception')->change();
+        });
+    }
+};
diff --git a/lang/en_US/import.php b/lang/en_US/import.php
@@ -1,14 +1,17 @@
 <?php
 return [
     'import' => 'Import',
+    'import_queue' => 'Import Queue',
+    'failed_imports' => 'Failed Imports',
+    'scheduled_for' => 'Scheduled for',
     'start_import' => 'Start Import',
     'import_running' => 'Import running...',
     'import_file' => 'File for Import',
 
-    'import_help' => 'You can import your existing browser bookmarks here. Usually, bookmarks are exported into an .html file by your browser. Select the file here and start the import.<br>Depending on the number of bookmarks this process may take some time.',
+    'import_help' => 'You can import your existing browser bookmarks here. Usually, bookmarks are exported into an .html file by your browser. Select the file here and start the import. Please note that a cron must be configured for the import to work.',
 
     'import_networkerror' => 'Something went wrong while trying to import the bookmarks. Please check your browser console for details or consult the application logs.',
     'import_error' => 'Something went wrong while trying to import the bookmarks. Please consult the application logs.',
     'import_empty' => 'Could not import any bookmarks. Either the uploaded file is corrupt or empty.',
-    'import_successfully' => ':imported links imported successfully, :skipped skipped. All imported links have been assigned the tag :taglink.',
+    'import_successfully' => ':queued links are queued for import and will be processed consecutively. :skipped links were skipped because they already exist in the database. All imported links will be assigned the tag :taglink.',
 ];
diff --git a/phpunit.xml b/phpunit.xml
@@ -43,7 +43,6 @@
     <server name="BCRYPT_ROUNDS" value="4"/>
     <server name="CACHE_DRIVER" value="array"/>
     <server name="MAIL_DRIVER" value="array"/>
-    <server name="QUEUE_CONNECTION" value="sync"/>
     <server name="SESSION_DRIVER" value="array"/>
     <server name="AUDIT_CONSOLE_EVENTS" value="true"/>
     <server name="APP_CONFIG_CACHE" value="bootstrap/cache/config.phpunit.php"/>