diff --git a/app/Actions/ImportHtmlBookmarks.php b/app/Actions/ImportHtmlBookmarks.php
index 8c0bc8425..212612de4 100644
--- a/app/Actions/ImportHtmlBookmarks.php
+++ b/app/Actions/ImportHtmlBookmarks.php
@@ -3,17 +3,15 @@
namespace App\Actions;
use App\Enums\ModelAttribute;
-use App\Helper\HtmlMeta;
-use App\Helper\LinkIconMapper;
+use App\Jobs\ImportLinkJob;
use App\Models\Link;
use App\Models\Tag;
-use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Log;
use Shaarli\NetscapeBookmarkParser\NetscapeBookmarkParser;
class ImportHtmlBookmarks
{
- protected int $imported = 0;
+ protected int $queued = 0;
protected int $skipped = 0;
protected ?Tag $importTag = null;
@@ -34,63 +32,23 @@ public function run(string $data, string $userId, bool $generateMeta = true): bo
'visibility' => ModelAttribute::VISIBILITY_PRIVATE,
]);
- foreach ($links as $link) {
+ foreach ($links as $i => $link) {
if (Link::whereUrl($link['url'])->first()) {
$this->skipped++;
continue;
}
- if ($generateMeta) {
- $linkMeta = (new HtmlMeta)->getFromUrl($link['url']);
- $title = $link['name'] ?: $linkMeta['title'];
- $description = $link['description'] ?: $linkMeta['description'];
- } else {
- $title = $link['name'];
- $description = $link['description'];
- }
-
- if (isset($link['public'])) {
- $visibility = $link['public'] ? ModelAttribute::VISIBILITY_PUBLIC : ModelAttribute::VISIBILITY_PRIVATE;
- } else {
- $visibility = usersettings('links_default_visibility');
- }
- $newLink = new Link([
- 'user_id' => $userId,
- 'url' => $link['url'],
- 'title' => $title,
- 'description' => $description,
- 'icon' => LinkIconMapper::getIconForUrl($link['url']),
- 'visibility' => $visibility,
- ]);
- $newLink->created_at = $link['dateCreated']
- ? Carbon::createFromTimestamp($link['dateCreated'])
- : Carbon::now();
- $newLink->updated_at = Carbon::now();
- $newLink->timestamps = false;
- $newLink->save();
-
- $newTags = [$this->importTag->id];
- if (!empty($link['tags'])) {
- foreach ($link['tags'] as $tag) {
- $newTag = Tag::firstOrCreate([
- 'user_id' => $userId,
- 'name' => $tag,
- 'visibility' => usersettings('tags_default_visibility'),
- ]);
- $newTags[] = $newTag->id;
- }
- }
- $newLink->tags()->sync($newTags);
+ dispatch(new ImportLinkJob($userId, $link, $this->importTag, $generateMeta))->delay($i * 10);
- $this->imported++;
+ $this->queued++;
}
return true;
}
- public function getImportCount(): int
+ public function getQueuedCount(): int
{
- return $this->imported;
+ return $this->queued;
}
public function getSkippedCount(): int
diff --git a/app/Console/Commands/ImportCommand.php b/app/Console/Commands/ImportCommand.php
index d855c095d..4f5708096 100644
--- a/app/Console/Commands/ImportCommand.php
+++ b/app/Console/Commands/ImportCommand.php
@@ -13,18 +13,17 @@ class ImportCommand extends Command
protected $signature = 'links:import
{filepath : Bookmarks file to import, use absolute paths if stored outside of LinkAce}
- {--skip-meta-generation : Whether the automatic generation of titles should be skipped.}
- {--skip-check : Whether the links checking should be skipped afterwards}';
+ {--skip-meta-generation : Whether the automatic generation of titles should be skipped.}';
protected $description = 'Import links from a bookmarks file which is stored locally in your file system.';
public function handle(): void
{
- $lookupMeta = true;
+ $generateMeta = true;
if ($this->option('skip-meta-generation')) {
$this->info('Skipping automatic meta generation.');
- $lookupMeta = false;
+ $generateMeta = false;
}
$this->info('You will be asked to select a user who will be the owner of the imported bookmarks now.');
@@ -40,24 +39,19 @@ public function handle(): void
}
$importer = new ImportHtmlBookmarks;
- $result = $importer->run($data, $this->user->id, $lookupMeta);
+ $result = $importer->run($data, $this->user->id, $generateMeta);
if ($result === false) {
$this->error('Error while importing bookmarks. Please check the application logs.');
return;
}
- if ($this->option('skip-check')) {
- $this->info('Skipping link check.');
- } elseif (config('mail.host') !== null) {
- Artisan::queue('links:check');
- } else {
- $this->warn('Links are configured to be checked, but email is not configured!');
- }
-
+ $tag = $importer->getImportTag();
$this->info(trans('import.import_successfully', [
- 'imported' => $importer->getImportCount(),
+ 'queued' => $importer->getQueuedCount(),
'skipped' => $importer->getSkippedCount(),
+
+ 'taglink' => sprintf('%s', route('tags.show', ['tag' => $tag]), $tag->name),
]));
}
}
diff --git a/app/Console/Commands/RegisterUserCommand.php b/app/Console/Commands/RegisterUserCommand.php
index 7c4301fd1..f40f4eae4 100644
--- a/app/Console/Commands/RegisterUserCommand.php
+++ b/app/Console/Commands/RegisterUserCommand.php
@@ -3,12 +3,13 @@
namespace App\Console\Commands;
use App\Actions\Fortify\CreateNewUser;
+use App\Enums\Role;
use Illuminate\Console\Command;
use Illuminate\Validation\ValidationException;
class RegisterUserCommand extends Command
{
- protected $signature = 'registeruser {name? : Username} {email? : User email address}';
+ protected $signature = 'registeruser {name? : Username} {email? : User email address} {--admin}';
protected $description = 'Register a new user with a given user name and an email address.';
private ?string $userName;
@@ -25,7 +26,7 @@ public function handle(): void
$this->askForUserDetails();
try {
- (new CreateNewUser)->create([
+ $newUser = (new CreateNewUser)->create([
'name' => $this->userName,
'email' => $this->userEmail,
'password' => $this->userPassword,
@@ -41,6 +42,10 @@ public function handle(): void
}
} while ($this->validationFailed);
+ if ($this->option('admin')) {
+ $newUser->assignRole(Role::ADMIN);
+ }
+
$this->info('User ' . $this->userName . ' registered.');
}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 613381e49..d100355fb 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -15,23 +15,11 @@
class Kernel extends ConsoleKernel
{
- protected $commands = [
- CheckLinksCommand::class,
- CompleteSetupCommand::class,
- ImportCommand::class,
- RegisterUserCommand::class,
- ResetPasswordCommand::class,
- UpdateLinkThumbnails::class,
- ListUsersCommand::class,
- ViewRecoveryCodesCommand::class,
- ];
-
protected function schedule(Schedule $schedule): void
{
- $schedule->command('links:check')->hourly();
+ $schedule->command('queue:work --queue=default,import')->withoutOverlapping();
- $schedule->command('queue:work --daemon --once')
- ->withoutOverlapping();
+ $schedule->command('links:check')->hourly();
if (config('backup.backup.enabled')) {
$schedule->command('backup:clean')->daily()->at('01:00');
diff --git a/app/Http/Controllers/App/ImportController.php b/app/Http/Controllers/App/ImportController.php
index 16e6f25c9..246a4a238 100644
--- a/app/Http/Controllers/App/ImportController.php
+++ b/app/Http/Controllers/App/ImportController.php
@@ -5,19 +5,38 @@
use App\Actions\ImportHtmlBookmarks;
use App\Http\Controllers\Controller;
use App\Http\Requests\DoImportRequest;
+use App\Jobs\ImportLinkJob;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Contracts\View\View;
use Illuminate\Http\JsonResponse;
+use Illuminate\Support\Facades\DB;
class ImportController extends Controller
{
- public function getImport(): View
+ public function form(): View
{
return view('app.import.import', [
'pageTitle' => trans('import.import'),
]);
}
+ public function queue(): View
+ {
+ $jobs = DB::table('jobs')
+ ->where('payload', 'LIKE', '%"displayName":' . json_encode(ImportLinkJob::class) . '%')
+ ->paginate(50);
+
+ $failedJobs = DB::table('failed_jobs')
+ ->where('payload', 'LIKE', '%"displayName":' . json_encode(ImportLinkJob::class) . '%')
+ ->paginate(50);
+
+ return view('app.import.queue', [
+ 'pageTitle' => trans('import.import'),
+ 'jobs' => $jobs,
+ 'failed_jobs' => $failedJobs,
+ ]);
+ }
+
/**
* Load the provided HTML bookmarks file and save all parsed results as new
* links including tags. This method is called via an Ajax call to prevent
@@ -42,12 +61,13 @@ public function doImport(DoImportRequest $request): JsonResponse
}
$tag = $importer->getImportTag();
+
return response()->json([
'success' => true,
'message' => trans('import.import_successfully', [
- 'imported' => $importer->getImportCount(),
+ 'queued' => $importer->getQueuedCount(),
'skipped' => $importer->getSkippedCount(),
- 'taglink' => sprintf('%s', route('tags.show', [$tag]), $tag->name),
+ 'taglink' => sprintf('%s', route('tags.show', ['tag' => $tag]), $tag->name),
]),
]);
}
diff --git a/app/Jobs/ImportLinkJob.php b/app/Jobs/ImportLinkJob.php
new file mode 100644
index 000000000..eeb039ff5
--- /dev/null
+++ b/app/Jobs/ImportLinkJob.php
@@ -0,0 +1,78 @@
+onQueue('import');
+ }
+
+ public function handle(): void
+ {
+ if ($this->generateMeta) {
+ $linkMeta = (new HtmlMeta)->getFromUrl($this->link['url']);
+ $title = $this->link['name'] ?: $linkMeta['title'];
+ $description = $this->link['description'] ?: $linkMeta['description'];
+ } else {
+ $title = $this->link['name'];
+ $description = $this->link['description'];
+ }
+
+ if (isset($this->link['public'])) {
+ $visibility = $this->link['public'] ? ModelAttribute::VISIBILITY_PUBLIC : ModelAttribute::VISIBILITY_PRIVATE;
+ } else {
+ $visibility = usersettings('links_default_visibility', $this->userId);
+ }
+
+ $newLink = new Link([
+ 'user_id' => $this->userId,
+ 'url' => $this->link['url'],
+ 'title' => $title,
+ 'description' => $description,
+ 'icon' => LinkIconMapper::getIconForUrl($this->link['url']),
+ 'visibility' => $visibility,
+ ]);
+
+ $newLink->created_at = $this->link['dateCreated']
+ ? Carbon::createFromTimestamp($this->link['dateCreated'])
+ : Carbon::now();
+ $newLink->updated_at = Carbon::now();
+ $newLink->timestamps = false;
+ $newLink->save();
+
+ $newTags = [$this->importTag->id];
+
+ if (!empty($this->link['tags'])) {
+ foreach ($this->link['tags'] as $tag) {
+ $newTag = Tag::firstOrCreate([
+ 'user_id' => $this->userId,
+ 'name' => $tag,
+ 'visibility' => usersettings('tags_default_visibility', $this->userId),
+ ]);
+ $newTags[] = $newTag->id;
+ }
+ }
+
+ $newLink->tags()->sync($newTags);
+ }
+}
diff --git a/database/migrations/2024_10_06_211654_update_failed_jobs_table.php b/database/migrations/2024_10_06_211654_update_failed_jobs_table.php
new file mode 100644
index 000000000..6c7c439e1
--- /dev/null
+++ b/database/migrations/2024_10_06_211654_update_failed_jobs_table.php
@@ -0,0 +1,25 @@
+string('uuid')->unique()->after('id');
+ $table->longText('payload')->change();
+ $table->longText('exception')->change();
+ });
+ }
+
+ public function down(): void
+ {
+ Schema::table('failed_jobs', function (Blueprint $table) {
+ $table->dropColumn(['uuid']);
+ $table->text('payload')->change();
+ $table->text('exception')->change();
+ });
+ }
+};
diff --git a/lang/en_US/import.php b/lang/en_US/import.php
index 0681cf008..e00767ebf 100644
--- a/lang/en_US/import.php
+++ b/lang/en_US/import.php
@@ -1,14 +1,17 @@
'Import',
+ 'import_queue' => 'Import Queue',
+ 'failed_imports' => 'Failed Imports',
+ 'scheduled_for' => 'Scheduled for',
'start_import' => 'Start Import',
'import_running' => 'Import running...',
'import_file' => 'File for Import',
- 'import_help' => 'You can import your existing browser bookmarks here. Usually, bookmarks are exported into an .html file by your browser. Select the file here and start the import.
Depending on the number of bookmarks this process may take some time.',
+ 'import_help' => 'You can import your existing browser bookmarks here. Usually, bookmarks are exported into an .html file by your browser. Select the file here and start the import. Please note that a cron must be configured for the import to work.',
'import_networkerror' => 'Something went wrong while trying to import the bookmarks. Please check your browser console for details or consult the application logs.',
'import_error' => 'Something went wrong while trying to import the bookmarks. Please consult the application logs.',
'import_empty' => 'Could not import any bookmarks. Either the uploaded file is corrupt or empty.',
- 'import_successfully' => ':imported links imported successfully, :skipped skipped. All imported links have been assigned the tag :taglink.',
+ 'import_successfully' => ':queued links are queued for import and will be processed consecutively. :skipped links were skipped because they already exist in the database. All imported links will be assigned the tag :taglink.',
];
diff --git a/phpunit.xml b/phpunit.xml
index 186138191..760f6480d 100644
--- a/phpunit.xml
+++ b/phpunit.xml
@@ -43,7 +43,6 @@
-
diff --git a/resources/views/app/import/queue.blade.php b/resources/views/app/import/queue.blade.php
new file mode 100644
index 000000000..5d5866b35
--- /dev/null
+++ b/resources/views/app/import/queue.blade.php
@@ -0,0 +1,89 @@
+@extends('layouts.app')
+
+@section('content')
+
+
+
+
+
+ @if($jobs->isNotEmpty())
+
+
+
+
+ ID |
+ @lang('link.url') |
+ @lang('import.scheduled_for') |
+
+
+
+ @foreach($jobs as $job)
+ @php
+ $data = unserialize(json_decode($job->payload)->data->command);
+ @endphp
+
+ {{ $job->id }} |
+ {{ $data->link['url'] }} |
+ {{ \Illuminate\Support\Carbon::parse($job->available_at) }} |
+
+ @endforeach
+
+
+
+
+ {{ $jobs->links() }}
+ @else
+
+
+ @lang('linkace.no_results_found', ['model' => trans('link.links')])
+
+
+ @endif
+
+
+
+
+
+
+
+ @if($failed_jobs->isNotEmpty())
+
+
+
+
+ ID |
+ @lang('link.url') |
+ |
+
+
+
+ @foreach($failed_jobs as $job)
+ @php
+ $data = unserialize(json_decode($job->payload)->data->command);
+ @endphp
+
+ {{ $job->id }} |
+ {{ $data->link['url'] }} |
+ {{ explode("\n", $job->exception)[0] ?? '' }} |
+
+ @endforeach
+
+
+
+
+ {{ $failed_jobs->links() }}
+ @else
+
+
+ @lang('linkace.no_results_found', ['model' => trans('link.links')])
+
+
+ @endif
+
+
+
+@endsection
diff --git a/resources/views/partials/nav-user.blade.php b/resources/views/partials/nav-user.blade.php
index eb7bdb3f9..face9d577 100644
--- a/resources/views/partials/nav-user.blade.php
+++ b/resources/views/partials/nav-user.blade.php
@@ -21,9 +21,12 @@
@lang('linkace.logout')
-
+
@lang('import.import')
+
+ @lang('import.import_queue')
+
@lang('export.export')
diff --git a/routes/web.php b/routes/web.php
index 2529a8a29..90e10a703 100644
--- a/routes/web.php
+++ b/routes/web.php
@@ -116,8 +116,10 @@
Route::post('search', [SearchController::class, 'doSearch'])
->name('do-search');
- Route::get('import', [ImportController::class, 'getImport'])
- ->name('get-import');
+ Route::get('import', [ImportController::class, 'form'])
+ ->name('import-form');
+ Route::get('import/queue', [ImportController::class, 'queue'])
+ ->name('import-queue');
Route::post('import', [ImportController::class, 'doImport'])
->name('do-import');
diff --git a/storage/app/.gitignore b/storage/app/.gitignore
index 1ea443919..d7c18057a 100644
--- a/storage/app/.gitignore
+++ b/storage/app/.gitignore
@@ -1,5 +1,6 @@
*
-!public/
!backup-temp/
!backups/
+!public/
+!temp/
!.gitignore
diff --git a/storage/app/temp/.gitignore b/storage/app/temp/.gitignore
new file mode 100644
index 000000000..d6b7ef32c
--- /dev/null
+++ b/storage/app/temp/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/tests/Controller/App/ImportControllerTest.php b/tests/Controller/App/ImportControllerTest.php
index 5d9454572..1bb670dad 100644
--- a/tests/Controller/App/ImportControllerTest.php
+++ b/tests/Controller/App/ImportControllerTest.php
@@ -3,25 +3,29 @@
namespace Tests\Controller\App;
use App\Enums\ModelAttribute;
+use App\Jobs\ImportLinkJob;
+use App\Models\Tag;
use App\Models\User;
use App\Settings\UserSettings;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Http\UploadedFile;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Http;
-use Illuminate\Testing\TestResponse;
+use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
class ImportControllerTest extends TestCase
{
use RefreshDatabase;
+ private User $user;
+
protected function setUp(): void
{
parent::setUp();
- $user = User::factory()->create();
- $this->actingAs($user);
+ $this->user = User::factory()->create();
+ $this->actingAs($this->user);
}
public function testValidImportResponse(): void
@@ -33,99 +37,254 @@ public function testValidImportResponse(): void
public function testValidImportActionResponse(): void
{
- $this->travelTo(Carbon::create(2024, 2, 20, 13, 16));
+ Queue::fake();
+
+ $exampleData = file_get_contents(__DIR__ . '/data/import_example.html');
+ $file = UploadedFile::fake()->createWithContent('import_example.html', $exampleData);
+
+ $response = $this->post('import', ['import-file' => $file], ['Accept' => 'application/json']);
+
+ $response->assertOk()->assertJson(['success' => true]);
+
+ Queue::assertPushed(ImportLinkJob::class, 5);
+ }
+
+ public function testQueuePage(): void
+ {
+ $exampleData = file_get_contents(__DIR__ . '/data/import_example.html');
+ $file = UploadedFile::fake()->createWithContent('import_example.html', $exampleData);
+ $response = $this->post('import', ['import-file' => $file], ['Accept' => 'application/json']);
+ $response->assertOk()->assertJson(['success' => true]);
+
+ $this->get('import/queue')->assertSeeInOrder([
+ 'https://medium.com/accelerated-intelligence',
+ 'https://adele.uxpin.com',
+ 'https://color.adobe.com/create/color-wheel',
+ 'https://loader.io',
+ 'https://astralapp.com',
+ ]);
+ }
+
+ public function testLinkImportJob(): void
+ {
UserSettings::fake([
'links_default_visibility' => ModelAttribute::VISIBILITY_INTERNAL,
'tags_default_visibility' => ModelAttribute::VISIBILITY_INTERNAL,
]);
- $response = $this->importBookmarks();
-
- $response->assertOk()
- ->assertJson([
- 'success' => true,
- ]);
+ $testHtml = 'DuckDuckGo