From ef2ee9abcc464556b5de00fec073052cac7240fb Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 20:03:03 +1000 Subject: [PATCH 001/109] Scaffolding second release --- .github/FUNDING.yml | 1 + .github/workflows/build.yml | 50 +-- .gitignore | 1 + .scrutinizer.yml | 2 + CHANGELOG.md | 2 +- README.md | 78 ++-- bootstrap.php | 10 + composer.json | 38 +- duster.json | 13 + helpers.php | 26 +- phpstan-baseline.neon | 1 + phpstan.neon | 6 + phpunit.xml.dist | 47 +-- pint.json | 19 + src/Concerns/HandlesTotalPages.php | 109 ----- src/Concerns/RetriesHttpRequests.php | 88 ---- src/Dtos/Config.php | 27 ++ src/Exceptions/LazyJsonPagesException.php | 12 +- src/Exceptions/OutOfAttemptsException.php | 51 --- src/Exceptions/UnsupportedSourceException.php | 16 + src/Handlers/AbstractHandler.php | 47 --- src/Handlers/ItemsPerPageHandler.php | 60 --- src/Handlers/LastPageHandler.php | 37 -- src/Handlers/NextPageHandler.php | 63 --- src/Handlers/TotalItemsHandler.php | 40 -- src/Handlers/TotalPagesHandler.php | 35 -- src/LazyJsonPages.php | 52 +++ src/Macro.php | 27 -- src/Outcome.php | 88 ---- src/Paginations/AnyPagination.php | 10 + src/Paginations/Pagination.php | 30 ++ .../LazyJsonPagesServiceProvider.php | 25 -- .../ConfigFactory.php} | 205 ++++------ src/Source.php | 79 ---- src/SourceWrapper.php | 111 ----- src/Sources/AnySource.php | 93 +++++ src/Sources/Source.php | 15 + tests/ConfigTest.php | 175 -------- tests/FixturesAware.php | 41 -- tests/LazyJsonPagesTest.php | 382 ------------------ tests/SourceWrapperTest.php | 97 ----- tests/fixtures/page1.json | 30 -- tests/fixtures/page2.json | 30 -- tests/fixtures/page3.json | 24 -- tests/fixtures/per_page.json | 18 - 45 files changed, 496 insertions(+), 1915 deletions(-) create mode 100644 bootstrap.php create mode 100644 duster.json create mode 100644 phpstan-baseline.neon create mode 100644 phpstan.neon create mode 100644 pint.json delete mode 100644 src/Concerns/HandlesTotalPages.php delete mode 100644 src/Concerns/RetriesHttpRequests.php create mode 100644 src/Dtos/Config.php delete mode 100644 src/Exceptions/OutOfAttemptsException.php create mode 100644 src/Exceptions/UnsupportedSourceException.php delete mode 100644 src/Handlers/AbstractHandler.php delete mode 100644 src/Handlers/ItemsPerPageHandler.php delete mode 100644 src/Handlers/LastPageHandler.php delete mode 100644 src/Handlers/NextPageHandler.php delete mode 100644 src/Handlers/TotalItemsHandler.php delete mode 100644 src/Handlers/TotalPagesHandler.php create mode 100644 src/LazyJsonPages.php delete mode 100644 src/Macro.php delete mode 100644 src/Outcome.php create mode 100644 src/Paginations/AnyPagination.php create mode 100644 src/Paginations/Pagination.php delete mode 100644 src/Providers/LazyJsonPagesServiceProvider.php rename src/{Config.php => Services/ConfigFactory.php} (55%) delete mode 100644 src/Source.php delete mode 100644 src/SourceWrapper.php create mode 100644 src/Sources/AnySource.php create mode 100644 src/Sources/Source.php delete mode 100644 tests/ConfigTest.php delete mode 100644 tests/FixturesAware.php delete mode 100644 tests/LazyJsonPagesTest.php delete mode 100644 tests/SourceWrapperTest.php delete mode 100644 tests/fixtures/page1.json delete mode 100644 tests/fixtures/page2.json delete mode 100644 tests/fixtures/page3.json delete mode 100644 tests/fixtures/per_page.json diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 0eb215b..af174ad 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,2 @@ github: cerbero90 +ko_fi: cerbero90 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c613d0..0845105 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,8 @@ name: build on: - push: - pull_request: + push: + pull_request: jobs: tests: @@ -11,20 +11,11 @@ jobs: strategy: fail-fast: false matrix: - php: [7.2, 7.3, 7.4, 8.0] - laravel: [6.*, 7.*, 8.*] + php: [8.1, 8.2, 8.3] dependency-version: [prefer-lowest, prefer-stable] - os: [ubuntu-latest, windows-latest] - exclude: - - laravel: 6.* - php: 8.0 - dependency-version: prefer-lowest - - laravel: 8.* - php: 7.2 - - os: windows-latest - php: 8.0 - - name: PHP ${{ matrix.php }} - Laravel ${{ matrix.laravel }} - ${{ matrix.dependency-version }} - ${{ matrix.os }} + os: [ubuntu-latest] + + name: PHP ${{ matrix.php }} - ${{ matrix.dependency-version }} - ${{ matrix.os }} steps: - name: Checkout code @@ -34,17 +25,15 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php }} - extensions: dom, curl, libxml, mbstring, zip tools: composer:v2 coverage: none - name: Install dependencies run: | - composer require "illuminate/contracts=${{ matrix.laravel }}" --no-interaction --no-update composer update --${{ matrix.dependency-version }} --prefer-dist --no-interaction - name: Execute tests - run: vendor/bin/phpunit --verbose + run: vendor/bin/pest coverage: runs-on: ubuntu-latest @@ -54,11 +43,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: 7.4 + php-version: 8.1 extensions: dom, curl, libxml, mbstring, zip tools: composer:v2 coverage: xdebug @@ -67,17 +58,16 @@ jobs: run: composer update --prefer-stable --prefer-dist --no-interaction - name: Execute tests - run: vendor/bin/phpunit --coverage-text --coverage-clover=coverage.clover + run: vendor/bin/pest --coverage-text --coverage-clover=coverage.clover - name: Upload coverage run: | - wget https://scrutinizer-ci.com/ocular.phar - php ocular.phar code-coverage:upload --format=php-clover coverage.clover + vendor/bin/ocular code-coverage:upload --format=php-clover coverage.clover - style: + linting: runs-on: ubuntu-latest - name: Coding style + name: Linting steps: - name: Checkout code @@ -86,9 +76,13 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: 8.0 - tools: phpcs + php-version: 8.2 + tools: composer:v2 coverage: none - - name: Execute check - run: phpcs --standard=psr12 src/ + - name: Install dependencies + run: | + composer update --prefer-stable --prefer-dist --no-interaction + + - name: Execute Duster + run: vendor/bin/duster lint -u tlint,phpcodesniffer,pint,phpstan -vvv diff --git a/.gitignore b/.gitignore index 7ae6add..bd2b70e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ vendor phpcs.xml phpunit.xml .phpunit.result.cache +.DS_Store diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 5d63b30..359494c 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,4 +1,6 @@ build: + environment: + php: 8.2 nodes: analysis: project_setup: diff --git a/CHANGELOG.md b/CHANGELOG.md index d837b00..234f4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to `lazy-json-pages` will be documented in this file. -Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) principles. +Updates should follow the [Keep a CHANGELOG](https://keepachangelog.com/) principles. ## 1.0.0 - 2021-08-08 diff --git a/README.md b/README.md index bc0eedd..21a7dbc 100644 --- a/README.md +++ b/README.md @@ -2,45 +2,44 @@ [![Author][ico-author]][link-author] [![PHP Version][ico-php]][link-php] -[![Laravel Version][ico-laravel]][link-laravel] -[![Octane Compatibility][ico-octane]][link-octane] [![Build Status][ico-actions]][link-actions] [![Coverage Status][ico-scrutinizer]][link-scrutinizer] [![Quality Score][ico-code-quality]][link-code-quality] +[![PHPStan Level][ico-phpstan]][link-phpstan] [![Latest Version][ico-version]][link-packagist] [![Software License][ico-license]](LICENSE.md) -[![PSR-7][ico-psr7]][link-psr7] -[![PSR-12][ico-psr12]][link-psr12] +[![PER][ico-per]][link-per] [![Total Downloads][ico-downloads]][link-downloads] -Framework agnostic package using asynchronous HTTP requests and generators to load paginated items of JSON APIs into [Laravel lazy collections](https://laravel.com/docs/collections#lazy-collections). +```php +$lazyCollection = LazyCollection::fromJsonPages($source, fn (Config $config) => $config + ->dot('data.results') + ->pages('total_pages') + ->perPage(500, 'page_size') + ->chunk(3) + ->timeout(15) + ->attempts(5) + ->backoff(fn (int $attempt) => $attempt ** 2 * 100)); +``` -Need to load heavy JSON with no pagination? Consider using [Lazy JSON](https://github.com/cerbero90/lazy-json) instead. +Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. +Need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. -## Install +## ๐Ÿ“ฆ Install -In a Laravel application, all you need to do is requiring the package: +Via Composer: ``` bash composer require cerbero/lazy-json-pages ``` -Otherwise, you also need to register the lazy collection macro: +## ๐Ÿ”ฎ Usage -``` php -use Cerbero\LazyJsonPages\Macro; -use Illuminate\Support\LazyCollection; - -LazyCollection::macro('fromJsonPages', new Macro()); -``` - -## Usage - -- [Length-aware paginations](#length-aware-paginations) -- [Cursor and next-page paginations](#cursor-and-next-page-paginations) -- [Fine-tuning the pages fetching process](#fine-tuning-the-pages-fetching-process) -- [Handling errors](#handling-errors) +- [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) +- [โ†ช๏ธ Cursor and next-page paginations](#-cursor-and-next-page-paginations) +- [๐Ÿ›  Requests fine-tuning](#-requests-fine-tuning) +- [๐Ÿ’ข Errors handling](#-errors-handling) Loading paginated items of JSON APIs into a lazy collection is possible by calling the collection itself or the included helper: @@ -109,7 +108,7 @@ lazyJsonPages($source, $path, function (Config $config) { The configuration depends on the type of pagination. Various paginations are supported, including length-aware and cursor paginations. -### Length-aware paginations +### ๐Ÿ“ Length-aware paginations The term "length-aware" indicates all paginations that show at least one of the following numbers: - the total number of pages @@ -198,7 +197,7 @@ $config->lastPage('last_page_key')->perPage(500, 'page_size'); ``` -### Cursor and next-page paginations +### โ†ช๏ธ Cursor and next-page paginations Some APIs show only the number or cursor of the next page in all pages. We can tackle this kind of pagination by indicating the JSON key holding the next page: @@ -209,7 +208,7 @@ $config->nextPage('next_page_key'); The JSON key may hold a number, a cursor or a URL, Lazy JSON Pages supports all of them. -### Fine-tuning the pages fetching process +### ๐Ÿ›  Requests fine-tuning Lazy JSON Pages provides a number of settings to adjust the way HTTP requests are sent to fetch pages. For example pages can be requested in chunks, so that only a few streams are kept in memory at once: @@ -275,7 +274,7 @@ $items ``` -### Handling errors +### ๐Ÿ’ข Errors handling As seen above, we can mitigate potentially faulty HTTP requests with backoffs, timeouts and retries. When we reach the maximum number of attempts and a request keeps failing, an `OutOfAttemptsException` is thrown. @@ -296,56 +295,51 @@ try { } ``` - -## Change log +## ๐Ÿ“† Change log Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. -## Testing +## ๐Ÿงช Testing ``` bash composer test ``` -## Contributing +## ๐Ÿ’ž Contributing Please see [CONTRIBUTING](CONTRIBUTING.md) and [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) for details. -## Security +## ๐Ÿงฏ Security If you discover any security related issues, please email andrea.marco.sartori@gmail.com instead of using the issue tracker. -## Credits +## ๐Ÿ… Credits - [Andrea Marco Sartori][link-author] - [All Contributors][link-contributors] -## License +## โš–๏ธ License The MIT License (MIT). Please see [License File](LICENSE.md) for more information. [ico-author]: https://img.shields.io/static/v1?label=author&message=cerbero90&color=50ABF1&logo=twitter&style=flat-square [ico-php]: https://img.shields.io/packagist/php-v/cerbero/lazy-json-pages?color=%234F5B93&logo=php&style=flat-square -[ico-laravel]: https://img.shields.io/static/v1?label=laravel&message=%E2%89%A56.0&color=ff2d20&logo=laravel&style=flat-square -[ico-octane]: https://img.shields.io/static/v1?label=octane&message=compatible&color=ff2d20&logo=laravel&style=flat-square [ico-version]: https://img.shields.io/packagist/v/cerbero/lazy-json-pages.svg?label=version&style=flat-square -[ico-actions]: https://img.shields.io/github/workflow/status/cerbero90/lazy-json-pages/build?style=flat-square&logo=github +[ico-actions]: https://img.shields.io/github/actions/workflow/status/cerbero90/lazy-json-pages/build.yml?branch=master&style=flat-square&logo=github [ico-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square -[ico-psr7]: https://img.shields.io/static/v1?label=compliance&message=PSR-7&color=blue&style=flat-square -[ico-psr12]: https://img.shields.io/static/v1?label=compliance&message=PSR-12&color=blue&style=flat-square +[ico-per]: https://img.shields.io/static/v1?label=compliance&message=PER&color=blue&style=flat-square [ico-scrutinizer]: https://img.shields.io/scrutinizer/coverage/g/cerbero90/lazy-json-pages.svg?style=flat-square&logo=scrutinizer [ico-code-quality]: https://img.shields.io/scrutinizer/g/cerbero90/lazy-json-pages.svg?style=flat-square&logo=scrutinizer +[ico-phpstan]: https://img.shields.io/badge/level-max-success?style=flat-square&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAGb0lEQVR42u1Xe1BUZRS/y4Kg8oiR3FCCBUySESZBRCiaBnmEsOzeSzsg+KxYYO9dEEftNRqZjx40FRZkTpqmOz5S2LsXlEZBciatkQnHDGYaGdFy1EpGMHl/p/PdFlt2rk5O+J9n5nA/vtf5ned3lnlISpRhafBlLRLHCtJGVrB/ZBDsaw2lUqzReGAC46DstTYfnSCGUjaaDvgxACo6j3vUenNdImeRXqdnWV5az5rrnzeZznj8J+E5Ftsclhf3s4J4CS/oRx5Bvon8ZU65FGYQxAwcf85a7CeRz+C41THejueydCZ7AAK34nwv3kHP/oUKdOL4K7258fF7Cud427O48RQeGkIGJ77N8fZqlrcfRP4d/x90WQfHXLeBt9dTrSlwl3V65ynWLM1SEA2qbNQckbe4Xmww10Hmy3shid0CMcmlEJtSDsl5VZBdfAgMvI3uuR+moJqN6LaxmpsOBeLCDmTifCB92RcQmbAUJvtqALc5sQr8p86gYBCcFdBq9wOin7NQax6ewlB6rqLZHf23FP10y3lj6uJtEBg2HxiVCtzd3SEwMBCio6Nh9uzZ4O/vLwOZ4OUNM2NyIGPFrvuzBG//lRPs+VQ2k1ki+ePkd84bskz7YFpYgizEz88P8vPzYffu3dDS0gJNTU1QXV0NqampRK1WIwgfiE4qhOyig0rC+pCvK8QUoML7uJVHA5kcQUp3DSpqWjc3d/Dy8oKioiLo6uqCoaEhuHb1KvT09AAhBFpbW4lOpyMyyIBQSCmoUQLQzgniNvz+obB2HS2RwBgE6dOxCyJogmNkP2u1Wrhw4QJ03+iGrR9XEd3CTNBn6eCbo40wPDwMdXV1BF1DVG5qiEtboxSUP6J71+D3NwUAhLOIRQzm7lnnhYUv7QFv/yDZ/Lm5ubK2DVI9iZ8bR8JDtEB57lNzENQN6OjoIGlpabIVZsYaMTO+hrikRRA1JxmSX9hE7/sJtVyF38tKsUCVZxBhz9jI3wGT/QJlADzPAyXrnj0kInzGHQCRMyOg/ed2uHjxIuE4TgYQHq2DLJqumashY+lnsMC4GVC5do6XVuK9l+4SkN8y+GfYeVJn2g++U7QygPT0dBgYGIDvT58mnF5PQcjC83PzSF9fH7S1tZGEhAQZQOT8JaA317oIkM6jS8uVLSDzOQqg23Uh+MlkOf00Gg0cP34c+vv74URzM9n41gby/rvvkc7OThlATU3NCGYJUXt4QaLuTYwBcTSOBmj1RD7D4Tsix4ByOjZRF/zgupDEbgZ3j4ly/qekpND0o5aQ44HS4OAgsVqtI1gTZO01IbG0aP1bknnxCDUvArHi+B0lJSlzglTFYO2udF3Ql9TCrHn5oEIreHp6QlRUFJSUlJCqqipSWVlJ8vLyCGYIFS7HS3zGa87mv4lcjLwLlStlLTKYYUUAlvrlDGcW45wKxXX6aqHZNutM+1oQBHFTewAKkoH4+vqCj48PYAGS5yb5amjNoO+CU2SL53NKpDD0vxHHmOJir7L5xUvZgm0us2R142ScOIyVqYvlpWU4XoHIP8DXL2b+wjdWeXh6U2FjmIIKmbWAYPFRMus62h/geIvjOQYlpuDysQrLL6Ger49HgW8jqvXUhI7UvDb9iaSTDqHtyItiF5Suw5ewF/Nd8VJ6zlhsn06bEhwX4NyfCvuGEeRpTmh4mkG68yDpyuzB9EUcjU5awbAgncPlAeSdAQER0zCndzqVbeXC4qDsMpvGEYBXRnsDx4N3Auf1FCTjTIaVtY/QTmd0I8bBVm1kejEubUfO01vqImn3c49X7qpeqI9inIgtbpxK3YrKfIJCt+OeV2nfUVFR4ca4EkVENyA7gkYcMfB1R5MMmxZ7ez/2KF5SSN1yV+158UPsJT0ZBcI2bRLtIXGoYu5FerOUiJe1OfsL3XEWH43l2KS+iJF9+S4FpcNgsc+j8cT8H4o1bfPg/qkLt50uJ1RzdMsGg0UqwfEN114Pwb1CtWTGg+Y9U5ClK9x7xUWI7BI5VQVp0AVcQ3bZkQhmnEgdHhKyNSZe16crtBIlc7sIb6cRLft2PCgoKGjijBDtjrAQ7a3EdMsxzIRflAFIhPb6mHYmYwX+WBlPQgskhgVryyJCQyNyBLsBQdQ6fgsQhyt6MSOOsWZ7gbH8wETmgRKAijatNL8Ngm0xx4tLcsps0Wzx4al0jXlI40B/A3pa144MDtSgAAAAAElFTkSuQmCC [ico-downloads]: https://img.shields.io/packagist/dt/cerbero/lazy-json-pages.svg?style=flat-square [link-author]: https://twitter.com/cerbero90 [link-php]: https://www.php.net -[link-laravel]: https://laravel.com -[link-octane]: https://github.com/laravel/octane [link-packagist]: https://packagist.org/packages/cerbero/lazy-json-pages [link-actions]: https://github.com/cerbero90/lazy-json-pages/actions?query=workflow%3Abuild -[link-psr7]: https://www.php-fig.org/psr/psr-7/ -[link-psr12]: https://www.php-fig.org/psr/psr-12/ +[link-per]: https://www.php-fig.org/per/coding-style/ [link-scrutinizer]: https://scrutinizer-ci.com/g/cerbero90/lazy-json-pages/code-structure [link-code-quality]: https://scrutinizer-ci.com/g/cerbero90/lazy-json-pages [link-downloads]: https://packagist.org/packages/cerbero/lazy-json-pages +[link-phpstan]: https://phpstan.org/ [link-contributors]: ../../contributors diff --git a/bootstrap.php b/bootstrap.php new file mode 100644 index 0000000..c658bd3 --- /dev/null +++ b/bootstrap.php @@ -0,0 +1,10 @@ +=6.0" + "php": "^8.1", + "cerbero/lazy-json": "^2.0", + "guzzlehttp/guzzle": "^7.2" }, "require-dev": { - "illuminate/http": ">=6.0", + "illuminate/http": ">=6.20", "mockery/mockery": "^1.3.4", - "phpunit/phpunit": ">=8.0", - "squizlabs/php_codesniffer": "^3.0" + "orchestra/testbench": ">=7.0", + "pestphp/pest": "^2.0", + "phpstan/phpstan": "^1.9", + "scrutinizer/ocular": "^1.8", + "squizlabs/php_codesniffer": "^3.0", + "tightenco/duster": "^2.0" }, "autoload": { "psr-4": { "Cerbero\\LazyJsonPages\\": "src" }, "files": [ + "bootstrap.php", "helpers.php" ] }, @@ -45,21 +49,19 @@ } }, "scripts": { - "test": "phpunit", - "check-style": "phpcs --standard=PSR12 src", - "fix-style": "phpcbf --standard=PSR12 src" + "fix": "duster fix -u tlint,phpcodesniffer,pint", + "lint": "duster lint -u tlint,phpcodesniffer,pint,phpstan", + "test": "pest" }, "extra": { "branch-alias": { "dev-master": "1.0-dev" - }, - "laravel": { - "providers": [ - "Cerbero\\LazyJsonPages\\Providers\\LazyJsonPagesServiceProvider" - ] } }, "config": { - "sort-packages": true + "sort-packages": true, + "allow-plugins": { + "pestphp/pest-plugin": true + } } } diff --git a/duster.json b/duster.json new file mode 100644 index 0000000..e3834c7 --- /dev/null +++ b/duster.json @@ -0,0 +1,13 @@ +{ + "include": [ + "src" + ], + "exclude": [ + "tests" + ], + "scripts": { + "lint": { + "phpstan": ["./vendor/bin/phpstan", "analyse"] + } + } +} diff --git a/helpers.php b/helpers.php index 700f49f..26dd141 100644 --- a/helpers.php +++ b/helpers.php @@ -1,18 +1,18 @@ - - - - tests - - - - - src/ - - - - - - - - + + + + + + + + + + + tests + + + + + + + + src/ + + diff --git a/pint.json b/pint.json new file mode 100644 index 0000000..03e092d --- /dev/null +++ b/pint.json @@ -0,0 +1,19 @@ +{ + "preset": "per", + "rules": { + "align_multiline_comment": true, + "combine_consecutive_issets": true, + "combine_consecutive_unsets": true, + "concat_space": {"spacing": "one"}, + "explicit_string_variable": true, + "ordered_imports": { + "sort_algorithm": "alpha", + "imports_order": [ + "class", + "function", + "const" + ] + }, + "simple_to_complex_string_variable": true + } +} diff --git a/src/Concerns/HandlesTotalPages.php b/src/Concerns/HandlesTotalPages.php deleted file mode 100644 index 5a8ee82..0000000 --- a/src/Concerns/HandlesTotalPages.php +++ /dev/null @@ -1,109 +0,0 @@ -config->source->request->getUri(); - $firstPageAlreadyFetched = strval($uri) == strval($this->config->source->request->getUri()); - $chunkedPages = $this->chunkPages($pages, $firstPageAlreadyFetched); - $items = $this->fetchItemsAsynchronously($chunkedPages, $uri); - - if ($firstPageAlreadyFetched) { - yield from $this->config->source->json($this->config->path); - } - - yield from $items; - } - - /** - * Retrieve the given pages in chunks - * - * @param int $pages - * @param bool $skipFirstPage - * @return iterable - */ - protected function chunkPages(int $pages, bool $skipFirstPage): iterable - { - $firstPage = $skipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; - $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; - - return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->chunk ?: INF); - } - - /** - * Fetch items by performing asynchronous HTTP calls - * - * @param iterable $chunkedPages - * @param Uri $uri - * @return Traversable - */ - protected function fetchItemsAsynchronously(iterable $chunkedPages, Uri $uri): Traversable - { - $client = new Client(['timeout' => $this->config->timeout]); - - foreach ($chunkedPages as $pages) { - $outcome = $this->retry(function (Outcome $outcome) use ($uri, $client, $pages) { - $pages = $outcome->pullFailedPages() ?: $pages; - - return $this->pool($client, $outcome, function () use ($uri, $pages) { - $request = clone $this->config->source->request; - - foreach ($pages as $page) { - yield $page => $request->withUri(Uri::withQueryValue($uri, $this->config->pageName, $page)); - } - }); - }); - - yield from $outcome->pullItems(); - } - } - - /** - * Retrieve the outcome of a pool of asynchronous requests - * - * @param Client $client - * @param Outcome $outcome - * @param callable $getRequests - * @return Outcome - */ - protected function pool(Client $client, Outcome $outcome, callable $getRequests): Outcome - { - $pool = new Pool($client, $getRequests(), [ - 'concurrency' => $this->config->concurrency, - 'fulfilled' => function (ResponseInterface $response, int $page) use ($outcome) { - $outcome->addItemsFromPage($page, $response, $this->config->path); - }, - 'rejected' => function (Throwable $e, int $page) use ($outcome) { - $outcome->addFailedPage($page); - throw $e; - } - ]); - - $pool->promise()->wait(); - - return $outcome; - } -} diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php deleted file mode 100644 index f722170..0000000 --- a/src/Concerns/RetriesHttpRequests.php +++ /dev/null @@ -1,88 +0,0 @@ -config->attempts; - - do { - $attempt++; - $remainingAttempts--; - - try { - return $callback($outcome); - } catch (Throwable $e) { - if ($remainingAttempts > 0) { - $this->backoff($attempt); - } else { - throw new OutOfAttemptsException($e, $outcome); - } - } - } while ($remainingAttempts > 0); - } - - /** - * Execute the backoff strategy - * - * @param int $attempt - * @return void - */ - protected function backoff(int $attempt): void - { - $backoff = $this->config->backoff ?: function (int $attempt) { - return ($attempt - 1) ** 2 * 1000; - }; - - usleep($backoff($attempt) * 1000); - } - - /** - * Retry to yield the result of HTTP requests - * - * @param callable $callback - * @return mixed - */ - protected function retryYielding(callable $callback) - { - $attempt = 0; - $outcome = new Outcome(); - $remainingAttempts = $this->config->attempts; - - do { - $failed = false; - $attempt++; - $remainingAttempts--; - - try { - yield from $callback($outcome); - } catch (Throwable $e) { - $failed = true; - - if ($remainingAttempts > 0) { - $this->backoff($attempt); - } else { - throw new OutOfAttemptsException($e, $outcome); - } - } - } while ($failed && $remainingAttempts > 0); - } -} diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php new file mode 100644 index 0000000..761a4ff --- /dev/null +++ b/src/Dtos/Config.php @@ -0,0 +1,27 @@ +getMessage()); + } } diff --git a/src/Exceptions/OutOfAttemptsException.php b/src/Exceptions/OutOfAttemptsException.php deleted file mode 100644 index e178cbc..0000000 --- a/src/Exceptions/OutOfAttemptsException.php +++ /dev/null @@ -1,51 +0,0 @@ -message = $original->getMessage(); - $this->original = $original; - $this->failedPages = $outcome->pullFailedPages(); - $this->items = new LazyCollection(function () use ($outcome) { - yield from $outcome->pullItems(); - }); - } -} diff --git a/src/Exceptions/UnsupportedSourceException.php b/src/Exceptions/UnsupportedSourceException.php new file mode 100644 index 0000000..a9614f6 --- /dev/null +++ b/src/Exceptions/UnsupportedSourceException.php @@ -0,0 +1,16 @@ +config = $config; - } - - /** - * Determine whether the handler can handle the APIs configuration - * - * @return bool - */ - abstract public function matches(): bool; - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - abstract public function handle(): Traversable; -} diff --git a/src/Handlers/ItemsPerPageHandler.php b/src/Handlers/ItemsPerPageHandler.php deleted file mode 100644 index 77df0b2..0000000 --- a/src/Handlers/ItemsPerPageHandler.php +++ /dev/null @@ -1,60 +0,0 @@ -config->perPageQuery - && $this->config->nextPageKey === null - && ($this->config->items || $this->config->pages || $this->config->lastPage); - } - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - public function handle(): Traversable - { - $originalUri = $this->config->source->request->getUri(); - $pages = (int) ceil($this->countItems() / $this->config->perPageOverride); - $uri = Uri::withQueryValue($originalUri, $this->config->perPageQuery, $this->config->perPageOverride); - - yield from $this->handleByTotalPages($pages, $uri); - } - - /** - * Retrieve the total number of items - * - * @return int - */ - protected function countItems(): int - { - if ($this->config->items) { - return $this->config->items; - } elseif ($this->config->pages) { - return $this->config->pages * $this->config->perPage; - } - - $pages = $this->config->firstPage == 0 ? $this->config->lastPage + 1 : $this->config->lastPage; - - return $pages * $this->config->perPage; - } -} diff --git a/src/Handlers/LastPageHandler.php b/src/Handlers/LastPageHandler.php deleted file mode 100644 index 42a62ef..0000000 --- a/src/Handlers/LastPageHandler.php +++ /dev/null @@ -1,37 +0,0 @@ -config->lastPage > 0 && $this->config->perPageQuery === null; - } - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - public function handle(): Traversable - { - $pages = $this->config->firstPage == 0 ? $this->config->lastPage + 1 : $this->config->lastPage; - - yield from $this->handleByTotalPages($pages); - } -} diff --git a/src/Handlers/NextPageHandler.php b/src/Handlers/NextPageHandler.php deleted file mode 100644 index b83d498..0000000 --- a/src/Handlers/NextPageHandler.php +++ /dev/null @@ -1,63 +0,0 @@ -config->nextPageKey; - } - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - public function handle(): Traversable - { - yield from $this->retryYielding(function (Outcome $outcome) { - try { - yield from $this->handleByNextPage(); - } catch (Throwable $e) { - $outcome->pullFailedPages(); - $outcome->addFailedPage($this->config->nextPage); - throw $e; - } - }); - } - - /** - * Handle APIs with next page - * - * @return Traversable - */ - protected function handleByNextPage(): Traversable - { - $request = clone $this->config->source->request; - - yield from $this->config->source->json($this->config->path); - - while ($this->config->nextPage) { - $uri = Uri::withQueryValue($request->getUri(), $this->config->pageName, $this->config->nextPage); - $this->config->source = new SourceWrapper($request->withUri($uri)); - $this->config->nextPage($this->config->nextPageKey); - yield from $this->handleByNextPage(); - } - } -} diff --git a/src/Handlers/TotalItemsHandler.php b/src/Handlers/TotalItemsHandler.php deleted file mode 100644 index c59dbb7..0000000 --- a/src/Handlers/TotalItemsHandler.php +++ /dev/null @@ -1,40 +0,0 @@ -config->items > 0 - && $this->config->pages === null - && $this->config->perPageQuery === null; - } - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - public function handle(): Traversable - { - $perPage = $this->config->perPage ?? count($this->config->source->json($this->config->path)); - $pages = $perPage > 0 ? (int) ceil($this->config->items / $perPage) : 0; - - yield from $this->handleByTotalPages($pages); - } -} diff --git a/src/Handlers/TotalPagesHandler.php b/src/Handlers/TotalPagesHandler.php deleted file mode 100644 index 20968e3..0000000 --- a/src/Handlers/TotalPagesHandler.php +++ /dev/null @@ -1,35 +0,0 @@ -config->pages > 0 && $this->config->perPageQuery === null; - } - - /** - * Handle the APIs configuration - * - * @return Traversable - */ - public function handle(): Traversable - { - yield from $this->handleByTotalPages($this->config->pages); - } -} diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php new file mode 100644 index 0000000..c14e9dc --- /dev/null +++ b/src/LazyJsonPages.php @@ -0,0 +1,52 @@ + + */ +final class LazyJsonPages implements IteratorAggregate +{ + /** + * @param Closure(ConfigFactory): void $configure + * @return LazyCollection + */ + public static function from(mixed $source, Closure $configure): LazyCollection + { + $source = new AnySource($source); + $configure($config = new ConfigFactory($source)); + + return new LazyCollection(fn() => yield from new self($source, $config->make())); + } + + private function __construct( + private readonly AnySource $source, + private readonly Config $config, + ) { + } + + /** + * @return Traversable + */ + public function getIterator(): Traversable + { + try { + yield from new AnyPagination($this->source, $this->config); + } catch (Throwable $e) { + throw LazyJsonPagesException::from($e); + } + } +} diff --git a/src/Macro.php b/src/Macro.php deleted file mode 100644 index 5b9ef3c..0000000 --- a/src/Macro.php +++ /dev/null @@ -1,27 +0,0 @@ -items[$page] = (function () use ($response, $path) { - yield from new Source($response, $path); - })(); - - return $this; - } - - /** - * Traverse and unset the items - * - * @return Traversable - */ - public function pullItems(): Traversable - { - ksort($this->items); - - foreach ($this->items as $generator) { - yield from $generator; - } - - $this->items = []; - } - - /** - * Add the given page to the failed pages - * - * @param string|int $page - * @return self - */ - public function addFailedPage($page): self - { - $this->failedPages[] = $page; - - return $this; - } - - /** - * Retrieve and unset the failed pages - * - * @return array - */ - public function pullFailedPages(): array - { - $failedPages = $this->failedPages; - - $this->failedPages = []; - - return $failedPages; - } -} diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php new file mode 100644 index 0000000..fd117ff --- /dev/null +++ b/src/Paginations/AnyPagination.php @@ -0,0 +1,10 @@ + + */ +class pagination implements IteratorAggregate +{ + public final function __construct( + protected readonly AnySource $source, + protected readonly Config $config, + ) { + } + + /** + * @return Traversable + */ + public function getIterator(): Traversable + { + yield 1; + } +} diff --git a/src/Providers/LazyJsonPagesServiceProvider.php b/src/Providers/LazyJsonPagesServiceProvider.php deleted file mode 100644 index f5953ad..0000000 --- a/src/Providers/LazyJsonPagesServiceProvider.php +++ /dev/null @@ -1,25 +0,0 @@ -source = new SourceWrapper($source); - $this->path = $path; - $this->hydrateConfig($config); } /** - * Hydrate the configuration - * - * @param callable|array|string|int $config - * @return void - * - * @throws LazyJsonPagesException + * Set the dot-notation path to extract items from */ - protected function hydrateConfig($config): void + public function dot(string $dot): self { - if (is_callable($config)) { - $config($this); - } elseif (is_array($config)) { - $this->resolveConfig($config); - } elseif (is_string($config) || is_numeric($config)) { - $this->pages($config); - } else { - throw new LazyJsonPagesException('The provided configuration is not valid.'); - } - } + $this->dot = $dot; - /** - * Resolve the given configuration - * - * @param array $config - * @return void - * - * @throws LazyJsonPagesException - */ - protected function resolveConfig(array $config): void - { - foreach ($config as $key => $value) { - if (method_exists($this, $method = Str::camel($key))) { - $values = is_array($value) ? $value : [$value]; - call_user_func_array([$this, $method], $values); - } else { - throw new LazyJsonPagesException("The key [{$key}] is not valid."); - } - } + return $this; } /** - * Set the page name - * - * @param string $name - * @return self + * Set the name of the page */ public function pageName(string $name): self { @@ -216,17 +145,24 @@ public function firstPage(int $page): self /** * Set the total number of pages - * - * @param string|int $pages - * @return self */ - public function pages($pages): self + public function totalPages(Closure|string $totalPages): self { - $this->pages = $this->resolveInt($pages, 1); + $this->totalPages = $this->integerFromResponse($totalPages, minimum: 1); + + // $this->totalPages = $this->extractor->integerFromResponse($totalPages); return $this; } + private function integerFromResponse(Closure|string $key, int $minimum = 0): int + { + return (int) max($minimum, match (true) { + $key instanceof Closure => $key($this->source->response()), + default => $this->source->response()->json($key) ?? $this->source->response()->header($key), + }); + } + /** * Retrieve an integer from the given value * @@ -236,7 +172,7 @@ public function pages($pages): self */ protected function resolveInt($value, int $minimum): int { - return (int) max($minimum, $this->resolvePage($value)); + return max($minimum, (int) $this->resolvePage($value)); } /** @@ -259,39 +195,28 @@ protected function resolvePage($value) /** * Set the total number of items - * - * @param string|int $items - * @return self */ - public function items($items): self + public function totalItems(Closure|string $totalItems): self { - $this->items = $this->resolveInt($items, 0); + $this->totalItems = $this->integerFromResponse($totalItems); return $this; } /** * Set the number of items per page and optionally override it - * - * @param int $perPage - * @param string|null $query - * @param int $firstPageItems - * @return self */ - public function perPage(int $perPage, string $query = null, int $firstPageItems = 1): self + public function perPage(int $perPage, ?string $key = null, int $firstPageItems = 1): self { - $this->perPage = max(1, $query ? $firstPageItems : $perPage); - $this->perPageQuery = $query; - $this->perPageOverride = $query ? max(1, $perPage) : null; + $this->perPage = max(1, $key ? $firstPageItems : $perPage); + $this->perPageKey = $key; + $this->perPageOverride = $key ? max(1, $perPage) : null; return $this; } /** * Set the next page - * - * @param string $key - * @return self */ public function nextPage(string $key): self { @@ -301,6 +226,14 @@ public function nextPage(string $key): self return $this; } + private function pageFromResponse(Closure|string $key, int $minimum = 0): string|int + { + return (int) max($minimum, match (true) { + $key instanceof Closure => $key($this->source->response()), + default => $this->source->response()->json($key) ?? $this->source->response()->header($key), + }); + } + /** * Set the number of the last page * @@ -388,4 +321,26 @@ public function backoff(callable $callback): self return $this; } + + public function make(): Config + { + return new Config( + $this->dot, + $this->pageName, + $this->firstPage, + $this->totalPages, + $this->totalItems, + $this->perPage, + $this->perPageKey, + $this->perPageOverride, + $this->nextPage, + $this->nextPageKey, + $this->lastPage, + $this->chunk, + $this->concurrency, + $this->timeout, + $this->attempts, + $this->backoff, + ); + } } diff --git a/src/Source.php b/src/Source.php deleted file mode 100644 index cb61185..0000000 --- a/src/Source.php +++ /dev/null @@ -1,79 +0,0 @@ -traversable = $this->toTraversable(new Config($source, $path, $config)); - } - - /** - * Retrieve the traversable items depending on the given configuration - * - * @param Config $config - * @return Traversable - * - * @throws LazyJsonPagesException - */ - protected function toTraversable(Config $config): Traversable - { - foreach ($this->handlers as $class) { - /** @var Handlers\AbstractHandler $handler */ - $handler = new $class($config); - - if ($handler->matches()) { - return $handler->handle(); - } - } - - throw new LazyJsonPagesException('Unable to load paginated items from the provided source.'); - } - - /** - * Retrieve the traversable items - * - * @return Traversable - */ - public function getIterator(): Traversable - { - return $this->traversable; - } -} diff --git a/src/SourceWrapper.php b/src/SourceWrapper.php deleted file mode 100644 index eee7190..0000000 --- a/src/SourceWrapper.php +++ /dev/null @@ -1,111 +0,0 @@ -original = $source; - $this->request = $this->getSourceRequest(); - $this->response = $this->getSourceResponse(); - } - - /** - * Retrieve the HTTP request of the source - * - * @return RequestInterface - * - * @throws LazyJsonPagesException - */ - protected function getSourceRequest(): RequestInterface - { - if ($this->original instanceof RequestInterface) { - return $this->original; - } elseif (isset($this->original->transferStats)) { - return $this->original->transferStats->getRequest(); - } - - throw new LazyJsonPagesException('The HTTP client response is not aware of the original request.'); - } - - /** - * Retrieve the HTTP response of the source - * - * @return ResponseInterface - */ - protected function getSourceResponse(): ResponseInterface - { - if ($this->original instanceof RequestInterface) { - return (new Client())->send($this->original); - } - - return $this->original->toPsrResponse(); - } - - /** - * Retrieve a fragment of the decoded JSON - * - * @param string $path - * @param mixed $default - * @return mixed - */ - public function json(string $path, $default = null) - { - if (!isset($this->decodedJson)) { - $this->decodedJson = json_decode((string) $this->response->getBody(), true); - } - - return Arr::get($this->decodedJson, $path, $default); - } -} diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php new file mode 100644 index 0000000..d341b21 --- /dev/null +++ b/src/Sources/AnySource.php @@ -0,0 +1,93 @@ +[] + */ + protected array $supportedSources = [ + CustomSource::class, + Endpoint::class, + LaravelClientResponse::class, + Psr7Request::class, + ]; + + /** + * The matching source. + * + * @var Source|null + */ + protected ?Source $matchingSource; + + /** + * Retrieve the JSON fragments + * + * @return Traversable + * @throws UnsupportedSourceException + */ + public function getIterator(): Traversable + { + return $this->matchingSource(); + } + + /** + * Retrieve the matching source + * + * @return Source + * @throws UnsupportedSourceException + */ + protected function matchingSource(): Source + { + if (isset($this->matchingSource)) { + return $this->matchingSource; + } + + foreach ($this->sources() as $source) { + if ($source->matches()) { + return $this->matchingSource = $source; + } + } + + throw new UnsupportedSourceException($this->source); + } + + /** + * Retrieve all available sources + * + * @return Generator + */ + protected function sources(): Generator + { + foreach ($this->supportedSources as $source) { + yield new $source($this->source, $this->config); + } + } + + /** + * Determine whether the JSON source can be handled + * + * @return bool + */ + public function matches(): bool + { + return true; + } + + /** + * Retrieve the calculated size of the JSON source + * + * @return int|null + */ + protected function calculateSize(): ?int + { + return $this->matchingSource()->size(); + } +} diff --git a/src/Sources/Source.php b/src/Sources/Source.php new file mode 100644 index 0000000..e3b0251 --- /dev/null +++ b/src/Sources/Source.php @@ -0,0 +1,15 @@ +shouldReceive('json')->with('next_page')->andReturn(2); - $wrapper->shouldReceive('json')->with('last_page')->andReturn('https://paginated-json-api.test?page_name=3'); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - $config = new Config($source, 'path', function (Config $config) { - $config - ->pageName('page_name') - ->firstPage(0) - ->pages(123) - ->items(321) - ->perPage(200, 'per_page', 2) - ->nextPage('next_page') - ->lastPage('last_page') - ->chunk(3) - ->concurrency(4) - ->timeout(-1) - ->attempts(6) - ->backoff(function (int $attempt) { - return $attempt; - }); - }); - - $this->assertSame('path', $config->path); - $this->assertSame('page_name', $config->pageName); - $this->assertSame(0, $config->firstPage); - $this->assertSame(123, $config->pages); - $this->assertSame(321, $config->items); - $this->assertSame(2, $config->perPage); - $this->assertSame('per_page', $config->perPageQuery); - $this->assertSame(200, $config->perPageOverride); - $this->assertSame(2, $config->nextPage); - $this->assertSame('next_page', $config->nextPageKey); - $this->assertSame(3, $config->lastPage); - $this->assertSame(3, $config->chunk); - $this->assertSame(4, $config->concurrency); - $this->assertSame(0, $config->timeout); - $this->assertSame(6, $config->attempts); - $this->assertSame(7, ($config->backoff)(7)); - } - - /** - * @test - */ - public function sets_options_through_associative_array() - { - $wrapper = Mockery::mock('overload:' . SourceWrapper::class); - $wrapper->shouldReceive('json')->with('next_page')->andReturn(2); - $wrapper->shouldReceive('json')->with('last_page')->andReturn('https://paginated-json-api.test?page_name=3'); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - $config = new Config($source, 'path', [ - 'pageName' => 'page_name', - 'first_page' => 0, - 'pages' => 123, - 'items' => 321, - 'perPage' => [200, 'per_page', 2], - 'nextPage' => 'next_page', - 'lastPage' => 'last_page', - 'sync' => true, - 'concurrency' => 4, - 'timeout' => -1, - 'attempts' => 6, - 'backoff' => function (int $attempt) { - return $attempt; - } - ]); - - $this->assertSame('path', $config->path); - $this->assertSame('page_name', $config->pageName); - $this->assertSame(0, $config->firstPage); - $this->assertSame(123, $config->pages); - $this->assertSame(321, $config->items); - $this->assertSame(2, $config->perPage); - $this->assertSame('per_page', $config->perPageQuery); - $this->assertSame(200, $config->perPageOverride); - $this->assertSame(2, $config->nextPage); - $this->assertSame('next_page', $config->nextPageKey); - $this->assertSame(3, $config->lastPage); - $this->assertSame(1, $config->chunk); - $this->assertSame(4, $config->concurrency); - $this->assertSame(0, $config->timeout); - $this->assertSame(6, $config->attempts); - $this->assertSame(7, ($config->backoff)(7)); - } - - /** - * @test - */ - public function sets_the_total_pages_with_an_integer() - { - Mockery::mock('overload:' . SourceWrapper::class); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - $config = new Config($source, 'path', 123); - - $this->assertSame(123, $config->pages); - } - - /** - * @test - */ - public function sets_the_total_pages_with_a_json_key() - { - $wrapper = Mockery::mock('overload:' . SourceWrapper::class); - $wrapper->shouldReceive('json')->with('total_pages')->andReturn(3); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - $config = new Config($source, 'path', 'total_pages'); - - $this->assertSame(3, $config->pages); - } - - /** - * @test - */ - public function fails_when_bad_configurations_are_provided() - { - Mockery::mock('overload:' . SourceWrapper::class); - - $this->expectExceptionObject(new LazyJsonPagesException('The provided configuration is not valid.')); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - new Config($source, 'path', new stdClass()); - } - - /** - * @test - */ - public function fails_when_a_bad_option_is_provided() - { - Mockery::mock('overload:' . SourceWrapper::class); - - $this->expectExceptionObject(new LazyJsonPagesException('The key [bad] is not valid.')); - - $source = new Request('GET', 'https://paginated-json-api.test'); - - new Config($source, 'path', ['bad' => true]); - } -} diff --git a/tests/FixturesAware.php b/tests/FixturesAware.php deleted file mode 100644 index c2e107d..0000000 --- a/tests/FixturesAware.php +++ /dev/null @@ -1,41 +0,0 @@ -resolve($this->fixture($fixture)); - }); - } -} diff --git a/tests/LazyJsonPagesTest.php b/tests/LazyJsonPagesTest.php deleted file mode 100644 index 061dfbe..0000000 --- a/tests/LazyJsonPagesTest.php +++ /dev/null @@ -1,382 +0,0 @@ - $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = 'meta.pagination.total_pages'; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * Assert that all items are correctly lazy-loaded via the given configuration - * - * @param array $initialRequest - * @param array $asyncRequests - * @param mixed $config - * @param array|null $expectedIds - * @return void - */ - protected function assertAllItemsAreLazyLoaded( - array $initialRequest, - array $asyncRequests, - $config, - array $expectedIds = null - ): void { - $source = new Request('GET', key($initialRequest)); - $client = Mockery::mock('overload:' . Client::class, ClientInterface::class); - - $client->shouldReceive('send')->with($source)->andReturn(reset($initialRequest)); - - foreach ($asyncRequests as $url => $promise) { - $client->shouldReceive('sendAsync') - ->withArgs(function (Request $request) use ($url) { - return $request->getUri() == $url; - }) - ->andReturn($promise); - } - - $index = 0; - $expectedIds = $expectedIds ?: range(1, 13); - - lazyJsonPages($source, 'data.results', $config)->each(function ($item) use (&$index, $expectedIds) { - $this->assertSame($expectedIds[$index], $item['id']); - $index++; - }); - } - - /** - * @test - */ - public function handles_total_items() - { - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = ['items' => 'meta.pagination.total_items']; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_total_items_with_per_page() - { - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'items' => 'meta.pagination.total_items', - 'per_page' => 5, - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_items_per_page_with_total_pages() - { - $initialRequest = ['https://paginated-json-api.test?page_size=1' => $this->fixture('per_page')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page_size=5&page=1' => $this->promiseFixture('page1'), - 'https://paginated-json-api.test?page_size=5&page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page_size=5&page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'pages' => 'meta.pagination.total_pages', - 'per_page' => [5, 'page_size'], - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_items_per_page_with_total_items() - { - $initialRequest = ['https://paginated-json-api.test?page_size=1' => $this->fixture('per_page')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page_size=5&page=1' => $this->promiseFixture('page1'), - 'https://paginated-json-api.test?page_size=5&page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page_size=5&page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'items' => 'meta.pagination.total_items', - 'per_page' => [5, 'page_size'], - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_items_per_page_with_last_page() - { - $initialRequest = ['https://paginated-json-api.test?page_size=1' => $this->fixture('per_page')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page_size=5&page=1' => $this->promiseFixture('page1'), - 'https://paginated-json-api.test?page_size=5&page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page_size=5&page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'last_page' => 'meta.pagination.last_page', - 'per_page' => [5, 'page_size'], - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_items_per_page_with_last_page_and_first_page_equal_to_0() - { - $initialRequest = ['https://paginated-json-api.test?page_size=1' => $this->fixture('per_page')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page_size=5&page=0' => $this->promiseFixture('page1'), - 'https://paginated-json-api.test?page_size=5&page=1' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page_size=5&page=2' => $this->promiseFixture('page3'), - ]; - $config = [ - 'first_page' => 0, - 'last_page' => 'meta.pagination.last_page', - 'per_page' => [5, 'page_size'], - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_last_page() - { - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'last_page' => 'meta.pagination.last_page', - ]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function handles_last_page_and_first_page_equal_to_0() - { - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=1' => $this->promiseFixture('page1'), - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = [ - 'first_page' => 0, - 'last_page' => 'meta.pagination.last_page', - ]; - $expectedIds = array_merge(range(1, 5), range(1, 13)); - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config, $expectedIds); - } - - /** - * @test - */ - public function handles_next_page() - { - $config = ['next_page' => 'meta.pagination.next_page']; - $source = new Request('GET', 'https://paginated-json-api.test'); - $client = Mockery::mock('overload:' . Client::class, ClientInterface::class); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test'; - }) - ->andReturn($this->fixture('page1')); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=2'; - }) - ->andReturn($this->fixture('page2')); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=3'; - }) - ->andReturn($this->fixture('page3')); - - $index = 0; - $expectedIds = range(1, 13); - - lazyJsonPages($source, 'data.results', $config)->each(function ($item) use (&$index, $expectedIds) { - $this->assertSame($expectedIds[$index], $item['id']); - $index++; - }); - } - - /** - * @test - */ - public function chunks_pages() - { - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $asyncRequests = [ - 'https://paginated-json-api.test?page=2' => $this->promiseFixture('page2'), - 'https://paginated-json-api.test?page=3' => $this->promiseFixture('page3'), - ]; - $config = ['items' => 'meta.pagination.total_items', 'chunk' => 1]; - - $this->assertAllItemsAreLazyLoaded($initialRequest, $asyncRequests, $config); - } - - /** - * @test - */ - public function fails_if_configuration_does_not_match_with_any_handler() - { - $this->expectException(LazyJsonPagesException::class); - $this->expectExceptionMessage('Unable to load paginated items from the provided source.'); - - $initialRequest = ['https://paginated-json-api.test' => $this->fixture('page1')]; - $config = []; - - $this->assertAllItemsAreLazyLoaded($initialRequest, [], $config); - } - - /** - * @test - */ - public function handles_failures() - { - $source = new Request('GET', 'https://paginated-json-api.test'); - $client = Mockery::mock('overload:' . Client::class, ClientInterface::class); - - $client->shouldReceive('send')->with($source)->andReturn($this->fixture('page1')); - - $client->shouldReceive('sendAsync') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=2'; - }) - ->andReturn($this->promiseFixture('page2')); - - $client->shouldReceive('sendAsync') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=3'; - }) - ->andReturn(new Promise(function () { - throw new Exception('foo'); - })); - - try { - lazyJsonPages($source, 'data.results', 'meta.pagination.total_pages')->each(function () { - // - }); - } catch (Throwable $e) { - $this->assertInstanceOf(OutOfAttemptsException::class, $e); - $this->assertSame('foo', $e->getMessage()); - $this->assertSame([3], $e->failedPages); - $this->assertSame(5, $e->items->count()); - } - } - - /** - * @test - */ - public function handles_next_page_failures() - { - $config = ['next_page' => 'meta.pagination.next_page']; - $source = new Request('GET', 'https://paginated-json-api.test'); - $client = Mockery::mock('overload:' . Client::class, ClientInterface::class); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test'; - }) - ->andReturn($this->fixture('page1')); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=2'; - }) - ->andReturn($this->fixture('page2')); - - $client->shouldReceive('send') - ->withArgs(function (Request $request) { - return $request->getUri() == 'https://paginated-json-api.test?page=3'; - }) - ->andThrow(new Exception('foo')); - - try { - lazyJsonPages($source, 'data.results', $config)->each(function () { - // - }); - } catch (Throwable $e) { - $this->assertInstanceOf(OutOfAttemptsException::class, $e); - $this->assertSame('foo', $e->getMessage()); - $this->assertSame([3], $e->failedPages); - $this->assertSame(0, $e->items->count()); - } - } -} diff --git a/tests/SourceWrapperTest.php b/tests/SourceWrapperTest.php deleted file mode 100644 index 1d26945..0000000 --- a/tests/SourceWrapperTest.php +++ /dev/null @@ -1,97 +0,0 @@ -expectExceptionObject(new LazyJsonPagesException('The provided JSON source is not valid.')); - - new SourceWrapper(123); - } - - /** - * @test - * @runInSeparateProcess - */ - public function sets_response_and_request_from_a_psr7_request() - { - $source = new Request('GET', 'https://paginated-json-api.test'); - $client = Mockery::mock('overload:' . Client::class, ClientInterface::class); - $response = $this->fixture('page1'); - - $client->shouldReceive('send')->with($source)->andReturn($response); - - $wrapper = new SourceWrapper($source); - - $this->assertSame($source, $wrapper->original); - $this->assertSame($source, $wrapper->request); - $this->assertSame($response, $wrapper->response); - } - - /** - * @test - */ - public function sets_response_and_request_from_a_laravel_http_client_response() - { - if (!class_exists(Response::class)) { - $this->markTestSkipped('The Laravel HTTP client response class is required for this test.'); - } - - $response = $this->fixture('page1'); - $source = new Response($response); - $request = new Request('GET', 'https://paginated-json-api.test'); - $source->transferStats = new TransferStats($request); - - $wrapper = new SourceWrapper($source); - - $this->assertSame($source, $wrapper->original); - $this->assertSame($request, $wrapper->request); - $this->assertSame($response, $wrapper->response); - } - - /** - * @test - */ - public function fails_if_laravel_response_does_not_have_transfer_stats() - { - if (!class_exists(Response::class)) { - $this->markTestSkipped('The Laravel HTTP client response class is required for this test.'); - } - - $this->expectException(LazyJsonPagesException::class); - $this->expectExceptionMessage('The HTTP client response is not aware of the original request.'); - - $source = new Response($this->fixture('page1')); - - new SourceWrapper($source); - } -} diff --git a/tests/fixtures/page1.json b/tests/fixtures/page1.json deleted file mode 100644 index 98dfae0..0000000 --- a/tests/fixtures/page1.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "data": { - "results": [ - { - "id": 1 - }, - { - "id": 2 - }, - { - "id": 3 - }, - { - "id": 4 - }, - { - "id": 5 - } - ] - }, - "meta": { - "pagination": { - "total_pages": 3, - "total_items": 13, - "last_page": 3, - "next_page": 2, - "next_page_url": "https://paginated-json-api.test?page=2" - } - } -} diff --git a/tests/fixtures/page2.json b/tests/fixtures/page2.json deleted file mode 100644 index 5db4935..0000000 --- a/tests/fixtures/page2.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "data": { - "results": [ - { - "id": 6 - }, - { - "id": 7 - }, - { - "id": 8 - }, - { - "id": 9 - }, - { - "id": 10 - } - ] - }, - "meta": { - "pagination": { - "total_pages": 3, - "total_items": 13, - "last_page": 3, - "next_page": 3, - "next_page_url": "https://paginated-json-api.test?page=3" - } - } -} diff --git a/tests/fixtures/page3.json b/tests/fixtures/page3.json deleted file mode 100644 index c291478..0000000 --- a/tests/fixtures/page3.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "data": { - "results": [ - { - "id": 11 - }, - { - "id": 12 - }, - { - "id": 13 - } - ] - }, - "meta": { - "pagination": { - "total_pages": 3, - "total_items": 13, - "last_page": 3, - "next_page": null, - "next_page_url": null - } - } -} diff --git a/tests/fixtures/per_page.json b/tests/fixtures/per_page.json deleted file mode 100644 index 6e8ad3a..0000000 --- a/tests/fixtures/per_page.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "data": { - "results": [ - { - "id": 1 - } - ] - }, - "meta": { - "pagination": { - "total_pages": 13, - "total_items": 13, - "last_page": 13, - "next_page": 2, - "next_page_url": "https://paginated-json-api.test?page=2" - } - } -} From faca87adc8cf38bca6ccf88b9af9ceee02f108cc Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:44:29 +1000 Subject: [PATCH 002/109] Remove final annotation --- src/Exceptions/UnsupportedSourceException.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Exceptions/UnsupportedSourceException.php b/src/Exceptions/UnsupportedSourceException.php index a9614f6..00dbba6 100644 --- a/src/Exceptions/UnsupportedSourceException.php +++ b/src/Exceptions/UnsupportedSourceException.php @@ -4,7 +4,7 @@ namespace Cerbero\LazyJsonPages\Exceptions; -final class UnsupportedSourceException extends LazyJsonPagesException +class UnsupportedSourceException extends LazyJsonPagesException { /** * @param mixed $source From 6c106d54fd40d12911e71a663ba91ee9d9d818b2 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:45:33 +1000 Subject: [PATCH 003/109] Add comments --- src/LazyJsonPages.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index c14e9dc..d1a8c46 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -16,11 +16,15 @@ use Traversable; /** + * The Lazy JSON Pages entry-point. + * * @implements IteratorAggregate */ final class LazyJsonPages implements IteratorAggregate { /** + * Instantiate the class statically. + * * @param Closure(ConfigFactory): void $configure * @return LazyCollection */ @@ -32,6 +36,9 @@ public static function from(mixed $source, Closure $configure): LazyCollection return new LazyCollection(fn() => yield from new self($source, $config->make())); } + /** + * Instantiate the class. + */ private function __construct( private readonly AnySource $source, private readonly Config $config, @@ -39,6 +46,8 @@ private function __construct( } /** + * Retrieve the paginated items lazily. + * * @return Traversable */ public function getIterator(): Traversable From 12c977dc5b1f5c1a843cb4d6659e015e9e7204b3 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:45:49 +1000 Subject: [PATCH 004/109] Make pagination abstract --- src/Paginations/Pagination.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 44dd669..42880f5 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -12,7 +12,7 @@ /** * @implements IteratorAggregate */ -class pagination implements IteratorAggregate +abstract class Pagination implements IteratorAggregate { public final function __construct( protected readonly AnySource $source, From 333facb6d6508a870ef8b8cc57b75ebda2a6de47 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:46:41 +1000 Subject: [PATCH 005/109] Gather data from responses --- src/Services/ConfigFactory.php | 9 ++++++--- src/Sources/Source.php | 14 ++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Services/ConfigFactory.php b/src/Services/ConfigFactory.php index 6e4a3da..4e5f3a7 100644 --- a/src/Services/ConfigFactory.php +++ b/src/Services/ConfigFactory.php @@ -4,12 +4,15 @@ namespace Cerbero\LazyJsonPages\Services; +use Cerbero\JsonParser\Concerns\DetectsEndpoints; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Sources\AnySource; use Closure; final class ConfigFactory { + use DetectsEndpoints; + /** * The dot to extract items from. */ @@ -159,7 +162,7 @@ private function integerFromResponse(Closure|string $key, int $minimum = 0): int { return (int) max($minimum, match (true) { $key instanceof Closure => $key($this->source->response()), - default => $this->source->response()->json($key) ?? $this->source->response()->header($key), + default => $this->source->response($key), }); } @@ -185,7 +188,7 @@ protected function resolvePage($value) { if (is_numeric($value)) { return (int) $value; - } elseif ($this->isEndpoint($value = $this->source->json($value))) { + } elseif (is_string($value) && $this->isEndpoint($value = $this->source->response($value))) { parse_str(parse_url($value, PHP_URL_QUERY), $query); $value = $query[$this->pageName]; } @@ -230,7 +233,7 @@ private function pageFromResponse(Closure|string $key, int $minimum = 0): string { return (int) max($minimum, match (true) { $key instanceof Closure => $key($this->source->response()), - default => $this->source->response()->json($key) ?? $this->source->response()->header($key), + default => $this->source->response($key), }); } diff --git a/src/Sources/Source.php b/src/Sources/Source.php index e3b0251..4a0f225 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -5,11 +5,17 @@ namespace Cerbero\LazyJsonPages\Sources; use Psr\Http\Message\RequestInterface; -use Psr\Http\Message\ResponseInterface; -interface Source +abstract class Source { - public function request(): RequestInterface; + public final function __construct( + protected readonly mixed $source, + ) { + } - public function response(): ResponseInterface; + abstract public function matches(): bool; + + abstract public function request(): RequestInterface; + + abstract public function response(?string $key = null): mixed; } From da798a5361e2c140567b3e82ee97ca3dba1975e0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:46:52 +1000 Subject: [PATCH 006/109] Update source --- src/Sources/AnySource.php | 46 +++++++++++++++------------------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index d341b21..b15c18b 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -6,9 +6,10 @@ use Cerbero\LazyJsonPages\Exceptions\UnsupportedSourceException; use Generator; -use Traversable; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; -class AnySource implements Source +class AnySource extends Source { /** * @var class-string[] @@ -16,32 +17,34 @@ class AnySource implements Source protected array $supportedSources = [ CustomSource::class, Endpoint::class, + LaravelClientRequest::class, LaravelClientResponse::class, + LaravelRequest::class, Psr7Request::class, + SymfonyRequest::class, ]; /** * The matching source. - * - * @var Source|null */ protected ?Source $matchingSource; /** - * Retrieve the JSON fragments - * - * @return Traversable - * @throws UnsupportedSourceException + * Determine whether the JSON source can be handled */ - public function getIterator(): Traversable + public function matches(): bool { - return $this->matchingSource(); + return true; + } + + public function request(): RequestInterface + { + return $this->matchingSource()->request(); } /** * Retrieve the matching source * - * @return Source * @throws UnsupportedSourceException */ protected function matchingSource(): Source @@ -67,27 +70,12 @@ protected function matchingSource(): Source protected function sources(): Generator { foreach ($this->supportedSources as $source) { - yield new $source($this->source, $this->config); + yield new $source($this->source); } } - /** - * Determine whether the JSON source can be handled - * - * @return bool - */ - public function matches(): bool - { - return true; - } - - /** - * Retrieve the calculated size of the JSON source - * - * @return int|null - */ - protected function calculateSize(): ?int + public function response(?string $key = null): mixed { - return $this->matchingSource()->size(); + return $this->matchingSource()->response($key); } } From dc3de4eaba23676903554cbf83d86e5d904ae7f1 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 21:58:36 +1000 Subject: [PATCH 007/109] Fix style --- src/Dtos/Config.php | 3 +-- src/LazyJsonPages.php | 3 +-- src/Paginations/Pagination.php | 5 ++--- src/Services/ConfigFactory.php | 4 +--- src/Sources/AnySource.php | 1 - src/Sources/Source.php | 5 ++--- 6 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 761a4ff..0282a8d 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -22,6 +22,5 @@ public function __construct( public readonly int $concurrency, public readonly int $timeout, public readonly int $attempts, - ) { - } + ) {} } diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index d1a8c46..f48bb2c 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -42,8 +42,7 @@ public static function from(mixed $source, Closure $configure): LazyCollection private function __construct( private readonly AnySource $source, private readonly Config $config, - ) { - } + ) {} /** * Retrieve the paginated items lazily. diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 42880f5..2c19a3e 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -14,11 +14,10 @@ */ abstract class Pagination implements IteratorAggregate { - public final function __construct( + final public function __construct( protected readonly AnySource $source, protected readonly Config $config, - ) { - } + ) {} /** * @return Traversable diff --git a/src/Services/ConfigFactory.php b/src/Services/ConfigFactory.php index 4e5f3a7..e915c9e 100644 --- a/src/Services/ConfigFactory.php +++ b/src/Services/ConfigFactory.php @@ -109,9 +109,7 @@ final class ConfigFactory */ private $backoff; - public function __construct(private AnySource $source) - { - } + public function __construct(private AnySource $source) {} /** * Set the dot-notation path to extract items from diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index b15c18b..5555752 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -7,7 +7,6 @@ use Cerbero\LazyJsonPages\Exceptions\UnsupportedSourceException; use Generator; use Psr\Http\Message\RequestInterface; -use Psr\Http\Message\ResponseInterface; class AnySource extends Source { diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 4a0f225..2bd858b 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -8,10 +8,9 @@ abstract class Source { - public final function __construct( + final public function __construct( protected readonly mixed $source, - ) { - } + ) {} abstract public function matches(): bool; From eb98da1a3fa995068b8c780a3bc1b5c9e14ed40d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 19 Dec 2023 22:30:12 +1000 Subject: [PATCH 008/109] Fix compatibility with PER standard --- phpcs.xml.dist | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/phpcs.xml.dist b/phpcs.xml.dist index 4774e69..db41854 100644 --- a/phpcs.xml.dist +++ b/phpcs.xml.dist @@ -10,5 +10,8 @@ - - \ No newline at end of file + + + + + From 682f952ce96a4cc4a3b01692581c988549317a26 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 28 Dec 2023 20:53:45 +1000 Subject: [PATCH 009/109] Update exception --- src/Exceptions/UnsupportedSourceException.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Exceptions/UnsupportedSourceException.php b/src/Exceptions/UnsupportedSourceException.php index 00dbba6..65984e9 100644 --- a/src/Exceptions/UnsupportedSourceException.php +++ b/src/Exceptions/UnsupportedSourceException.php @@ -4,13 +4,16 @@ namespace Cerbero\LazyJsonPages\Exceptions; +/** + * The exception to throw when the given source is not supported. + */ class UnsupportedSourceException extends LazyJsonPagesException { /** - * @param mixed $source + * Instantiate the class. */ public function __construct(public readonly mixed $source) { - parent::__construct('The provided source is not supported'); + parent::__construct('The provided source is not supported.'); } } From ff35519e5d16f75868cef3101743ecd382e65765 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 28 Dec 2023 20:54:11 +1000 Subject: [PATCH 010/109] Implement the aggragation of paginations --- .../UnsupportedPaginationException.php | 21 +++++++++ src/Paginations/AnyPagination.php | 45 ++++++++++++++++++- src/Paginations/Pagination.php | 22 +++++---- 3 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 src/Exceptions/UnsupportedPaginationException.php diff --git a/src/Exceptions/UnsupportedPaginationException.php b/src/Exceptions/UnsupportedPaginationException.php new file mode 100644 index 0000000..5cfb6ee --- /dev/null +++ b/src/Exceptions/UnsupportedPaginationException.php @@ -0,0 +1,21 @@ +[] + */ + protected array $supportedPaginations = [ + CursorPagination::class, + CustomPagination::class, + LengthAwarePagination::class, + LinkHeaderPagination::class, + OffsetPagination::class, + ]; + + /** + * Determine whether the configuration matches this pagination. + */ + public function matches(): bool + { + return true; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + foreach ($this->supportedPaginations as $class) { + $pagination = new $class($this->source, $this->config); + + if ($pagination->matches()) { + return $pagination; + } + } + + throw new UnsupportedPaginationException($this->config); + } } diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 2c19a3e..b66424a 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -10,20 +10,26 @@ use Traversable; /** + * The abstract implementation of a pagination. + * * @implements IteratorAggregate */ abstract class Pagination implements IteratorAggregate { - final public function __construct( - protected readonly AnySource $source, - protected readonly Config $config, - ) {} + /** + * Determine whether the configuration matches this pagination. + */ + abstract public function matches(): bool; /** + * Yield the paginated items. + * * @return Traversable */ - public function getIterator(): Traversable - { - yield 1; - } + abstract public function getIterator(): Traversable; + + final public function __construct( + protected readonly AnySource $source, + protected readonly Config $config, + ) {} } From 9c1aa88c00a827151a763c0ac603e7b550e0832f Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 2 Jan 2024 22:32:35 +1000 Subject: [PATCH 011/109] Update configuration --- src/Dtos/Config.php | 4 +- src/Services/ConfigFactory.php | 108 +++++++++------------------------ 2 files changed, 30 insertions(+), 82 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 0282a8d..5522151 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -4,6 +4,8 @@ namespace Cerbero\LazyJsonPages\Dtos; +use Closure; + final class Config { public function __construct( @@ -15,7 +17,7 @@ public function __construct( public readonly ?int $perPage, public readonly ?string $perPageKey, public readonly int $perPageOverride, - public readonly string|int $nextPage, + public readonly ?Closure $nextPage, public readonly string $nextPageKey, public readonly int $lastPage, public readonly int $chunk, diff --git a/src/Services/ConfigFactory.php b/src/Services/ConfigFactory.php index e915c9e..a05514d 100644 --- a/src/Services/ConfigFactory.php +++ b/src/Services/ConfigFactory.php @@ -4,15 +4,13 @@ namespace Cerbero\LazyJsonPages\Services; -use Cerbero\JsonParser\Concerns\DetectsEndpoints; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Sources\AnySource; +use Cerbero\LazyJsonPages\ValueObjects\Response; use Closure; final class ConfigFactory { - use DetectsEndpoints; - /** * The dot to extract items from. */ @@ -51,68 +49,55 @@ final class ConfigFactory /** * The new number of items per page. */ - private int $perPageOverride; + private ?int $perPageOverride = null; /** - * The next page of a simple or cursor pagination. - * - * @var string|int + * The next page number, link or cursor. */ - private $nextPage; + private ?Closure $nextPage = null; /** * The key holding the next page. - * - * @var string */ - private $nextPageKey; + private ?string $nextPageKey = null; /** * The number of the last page. - * - * @var int */ - private $lastPage; + private ?int $lastPage = null; /** - * The number of pages to fetch per chunk. - * - * @var int + * The number of pages to fetch asynchronously per chunk. */ - private $chunk; + private ?int $chunk = null; /** * The maximum number of concurrent async HTTP requests. - * - * @var int */ - private $concurrency = 10; + private int $concurrency = 10; /** * The timeout in seconds. - * - * @var int */ - private $timeout = 5; + private int $timeout = 5; /** * The number of attempts to fetch pages. - * - * @var int */ - private $attempts = 3; + private int $attempts = 3; /** * The backoff strategy. - * - * @var callable */ - private $backoff; + private ?Closure $backoff = null; + /** + * Instantiate the class. + */ public function __construct(private AnySource $source) {} /** - * Set the dot-notation path to extract items from + * Set the dot-notation path to extract items from. */ public function dot(string $dot): self { @@ -122,7 +107,7 @@ public function dot(string $dot): self } /** - * Set the name of the page + * Set the name of the page. */ public function pageName(string $name): self { @@ -133,9 +118,6 @@ public function pageName(string $name): self /** * Set the number of the first page - * - * @param int $page - * @return self */ public function firstPage(int $page): self { @@ -151,47 +133,23 @@ public function totalPages(Closure|string $totalPages): self { $this->totalPages = $this->integerFromResponse($totalPages, minimum: 1); - // $this->totalPages = $this->extractor->integerFromResponse($totalPages); - return $this; } - private function integerFromResponse(Closure|string $key, int $minimum = 0): int - { - return (int) max($minimum, match (true) { - $key instanceof Closure => $key($this->source->response()), - default => $this->source->response($key), - }); - } - /** - * Retrieve an integer from the given value - * - * @param string|int $value - * @param int $minimum - * @return int + * Retrieve an integer from the response */ - protected function resolveInt($value, int $minimum): int + private function integerFromResponse(Closure|string $key, int $minimum = 0): int { - return max($minimum, (int) $this->resolvePage($value)); + return (int) max($minimum, $this->valueFromResponse($key)); } /** - * Retrieve the page value from the given presumed URL - * - * @param mixed $value - * @return mixed + * Retrieve a value from the response */ - protected function resolvePage($value) + private function valueFromResponse(Closure|string $key): mixed { - if (is_numeric($value)) { - return (int) $value; - } elseif (is_string($value) && $this->isEndpoint($value = $this->source->response($value))) { - parse_str(parse_url($value, PHP_URL_QUERY), $query); - $value = $query[$this->pageName]; - } - - return is_numeric($value) ? (int) $value : $value; + return $key instanceof Closure ? $key($this->source->response()) : $this->source->response($key); } /** @@ -219,31 +177,19 @@ public function perPage(int $perPage, ?string $key = null, int $firstPageItems = /** * Set the next page */ - public function nextPage(string $key): self + public function nextPage(Closure|string $key): self { - $this->nextPageKey = $key; - $this->nextPage = $this->resolvePage($key); + $this->nextPage = $key instanceof Closure ? $key : fn(Response $response) => $response->get($key); return $this; } - private function pageFromResponse(Closure|string $key, int $minimum = 0): string|int - { - return (int) max($minimum, match (true) { - $key instanceof Closure => $key($this->source->response()), - default => $this->source->response($key), - }); - } - /** * Set the number of the last page - * - * @param string|int $page - * @return self */ - public function lastPage($page): self + public function lastPage(Closure|string $key): self { - $this->lastPage = $this->resolvePage($page); + $this->lastPage = $this->integerFromResponse($key); return $this; } From a835702a7c494a7b469dca6cf1b0bc060d022f52 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 2 Jan 2024 22:33:14 +1000 Subject: [PATCH 012/109] Improve docblocks --- src/Paginations/AnyPagination.php | 2 +- src/Sources/AnySource.php | 36 +++++++++++++++---------------- src/Sources/Source.php | 11 ++++++++++ 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 3a7e9c4..1541e74 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -26,7 +26,7 @@ class AnyPagination extends Pagination ]; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index 5555752..2c4b5b5 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -5,22 +5,24 @@ namespace Cerbero\LazyJsonPages\Sources; use Cerbero\LazyJsonPages\Exceptions\UnsupportedSourceException; -use Generator; use Psr\Http\Message\RequestInterface; +/** + * The aggregator of sources. + */ class AnySource extends Source { /** * @var class-string[] */ protected array $supportedSources = [ - CustomSource::class, + // CustomSource::class, Endpoint::class, - LaravelClientRequest::class, - LaravelClientResponse::class, - LaravelRequest::class, - Psr7Request::class, - SymfonyRequest::class, + // LaravelClientRequest::class, + // LaravelClientResponse::class, + // LaravelRequest::class, + // Psr7Request::class, + // SymfonyRequest::class, ]; /** @@ -29,13 +31,16 @@ class AnySource extends Source protected ?Source $matchingSource; /** - * Determine whether the JSON source can be handled + * Determine whether this class can handle the source */ public function matches(): bool { return true; } + /** + * Retrieve the HTTP request + */ public function request(): RequestInterface { return $this->matchingSource()->request(); @@ -52,7 +57,9 @@ protected function matchingSource(): Source return $this->matchingSource; } - foreach ($this->sources() as $source) { + foreach ($this->supportedSources as $class) { + $source = new $class($this->source); + if ($source->matches()) { return $this->matchingSource = $source; } @@ -62,17 +69,10 @@ protected function matchingSource(): Source } /** - * Retrieve all available sources + * Retrieve the HTTP response or part of it * - * @return Generator + * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) */ - protected function sources(): Generator - { - foreach ($this->supportedSources as $source) { - yield new $source($this->source); - } - } - public function response(?string $key = null): mixed { return $this->matchingSource()->response($key); diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 2bd858b..93a6ed4 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -12,9 +12,20 @@ final public function __construct( protected readonly mixed $source, ) {} + /** + * Determine whether this class can handle the source + */ abstract public function matches(): bool; + /** + * Retrieve the HTTP request + */ abstract public function request(): RequestInterface; + /** + * Retrieve the HTTP response or part of it + * + * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) + */ abstract public function response(?string $key = null): mixed; } From 7472d235bf63562b746f2ca8c9d2822d9dc4dc2a Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 2 Jan 2024 22:36:00 +1000 Subject: [PATCH 013/109] Implement the HTTP response value object --- src/ValueObjects/Response.php | 81 +++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/ValueObjects/Response.php diff --git a/src/ValueObjects/Response.php b/src/ValueObjects/Response.php new file mode 100644 index 0000000..7d197db --- /dev/null +++ b/src/ValueObjects/Response.php @@ -0,0 +1,81 @@ + $headers + */ + public readonly array $headers; + + /** + * Instantiate the class. + * + * @param array $headers + */ + public function __construct(public readonly string $json, array $headers) + { + $this->headers = $this->normalizeHeaders($headers); + } + + /** + * Normalize the given headers. + * + * @param array $headers + * @return array + */ + private function normalizeHeaders(array $headers): array + { + $normalizedHeaders = []; + + foreach ($headers as $name => $value) { + $normalizedHeaders[strtolower($name)] = $value; + } + + return $normalizedHeaders; + } + + /** + * Retrieve a value from the body or a header. + */ + public function get(string $key): mixed + { + return $this->hasHeader($key) ? $this->header($key) : $this->json($key); + } + + /** + * Determine whether the given header is set. + */ + public function hasHeader(string $header): bool + { + return isset($this->headers[strtolower($header)]); + } + + /** + * Retrieve the given header. + */ + public function header(string $header): ?string + { + return $this->headers[strtolower($header)] ?? null; + } + + /** + * Retrieve a value from the body. + */ + public function json(string $key): mixed + { + $array = JsonParser::parse($this->json)->pointer($key)->toArray(); + + return empty($array) ? null : current($array); + } +} From 2ed66d9bc50094203d9555446f61ebdb8bb9026d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 2 Jan 2024 23:00:28 +1000 Subject: [PATCH 014/109] Improve docblocks --- src/Sources/AnySource.php | 8 ++++---- src/Sources/Source.php | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index 2c4b5b5..21d6156 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -31,7 +31,7 @@ class AnySource extends Source protected ?Source $matchingSource; /** - * Determine whether this class can handle the source + * Determine whether this class can handle the source. */ public function matches(): bool { @@ -39,7 +39,7 @@ public function matches(): bool } /** - * Retrieve the HTTP request + * Retrieve the HTTP request. */ public function request(): RequestInterface { @@ -47,7 +47,7 @@ public function request(): RequestInterface } /** - * Retrieve the matching source + * Retrieve the matching source. * * @throws UnsupportedSourceException */ @@ -69,7 +69,7 @@ protected function matchingSource(): Source } /** - * Retrieve the HTTP response or part of it + * Retrieve the HTTP response or part of it. * * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) */ diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 93a6ed4..34dbcaa 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -6,6 +6,9 @@ use Psr\Http\Message\RequestInterface; +/** + * The abstract implementation of a source. + */ abstract class Source { final public function __construct( @@ -13,17 +16,17 @@ final public function __construct( ) {} /** - * Determine whether this class can handle the source + * Determine whether this class can handle the source. */ abstract public function matches(): bool; /** - * Retrieve the HTTP request + * Retrieve the HTTP request. */ abstract public function request(): RequestInterface; /** - * Retrieve the HTTP response or part of it + * Retrieve the HTTP response or part of it. * * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) */ From 0d2285a16616382afb8bd0cecc37b766b176e6f3 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 2 Jan 2024 23:00:37 +1000 Subject: [PATCH 015/109] Implement the endpoint source --- src/Sources/Endpoint.php | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/Sources/Endpoint.php diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php new file mode 100644 index 0000000..76b66f7 --- /dev/null +++ b/src/Sources/Endpoint.php @@ -0,0 +1,64 @@ +source instanceof UriInterface + || (is_string($this->source) && $this->isEndpoint($this->source)); + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return $this->request ??= new Request('GET', $this->source, [ + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + ]); + } + + /** + * Retrieve the HTTP response or part of it. + * + * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) + */ + public function response(?string $key = null): mixed + { + $this->response ??= (new Client())->send($this->request()); + + return $key === null ? $this->response : $this->response->get($key); + } +} From 746be5e610797caaa12d5c74fdf152681858a478 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 8 Jan 2024 17:27:09 +1000 Subject: [PATCH 016/109] Rename ConfigFactory to Api --- src/LazyJsonPages.php | 24 +++--- src/Services/{ConfigFactory.php => Api.php} | 94 ++++++++++----------- 2 files changed, 54 insertions(+), 64 deletions(-) rename src/Services/{ConfigFactory.php => Api.php} (72%) diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index f48bb2c..23bb0b6 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -5,35 +5,33 @@ namespace Cerbero\LazyJsonPages; use Cerbero\LazyJsonPages\Dtos\Config; -use Cerbero\LazyJsonPages\Exceptions\LazyJsonPagesException; use Cerbero\LazyJsonPages\Paginations\AnyPagination; -use Cerbero\LazyJsonPages\Services\ConfigFactory; +use Cerbero\LazyJsonPages\Services\Api; use Cerbero\LazyJsonPages\Sources\AnySource; use Closure; use Illuminate\Support\LazyCollection; use IteratorAggregate; -use Throwable; use Traversable; /** * The Lazy JSON Pages entry-point. * - * @implements IteratorAggregate + * @implements IteratorAggregate */ final class LazyJsonPages implements IteratorAggregate { /** * Instantiate the class statically. * - * @param Closure(ConfigFactory): void $configure - * @return LazyCollection + * @param Closure(Api): void $configure + * @return LazyCollection */ public static function from(mixed $source, Closure $configure): LazyCollection { $source = new AnySource($source); - $configure($config = new ConfigFactory($source)); + $configure($api = new Api($source)); - return new LazyCollection(fn() => yield from new self($source, $config->make())); + return new LazyCollection(fn() => yield from new self($source, $api->toConfig())); } /** @@ -47,14 +45,14 @@ private function __construct( /** * Retrieve the paginated items lazily. * - * @return Traversable + * @return Traversable */ public function getIterator(): Traversable { - try { - yield from new AnyPagination($this->source, $this->config); - } catch (Throwable $e) { - throw LazyJsonPagesException::from($e); + // yield each item within a loop - instead of using `yield from` - to ignore the actual item index + // and ensure indexes continuity, otherwise the index of items always starts from 0 on every page. + foreach (new AnyPagination($this->source, $this->config) as $item) { + yield $item; } } } diff --git a/src/Services/ConfigFactory.php b/src/Services/Api.php similarity index 72% rename from src/Services/ConfigFactory.php rename to src/Services/Api.php index a05514d..86030d5 100644 --- a/src/Services/ConfigFactory.php +++ b/src/Services/Api.php @@ -4,18 +4,24 @@ namespace Cerbero\LazyJsonPages\Services; +use Cerbero\LazyJson\Pointers\DotsConverter; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Sources\AnySource; use Cerbero\LazyJsonPages\ValueObjects\Response; use Closure; -final class ConfigFactory +final class Api { /** * The dot to extract items from. */ private string $dot = '*'; + /** + * The JSON pointer to extract items from. + */ + private string $pointer = ''; + /** * The name of the page. */ @@ -67,19 +73,19 @@ final class ConfigFactory private ?int $lastPage = null; /** - * The number of pages to fetch asynchronously per chunk. + * The maximum number of concurrent async HTTP requests. */ - private ?int $chunk = null; + private int $async = 3; /** - * The maximum number of concurrent async HTTP requests. + * The server connection timeout in seconds. */ - private int $concurrency = 10; + private int $connectionTimeout = 5; /** - * The timeout in seconds. + * The HTTP request timeout in seconds. */ - private int $timeout = 5; + private int $requestTimeout = 5; /** * The number of attempts to fetch pages. @@ -102,6 +108,7 @@ public function __construct(private AnySource $source) {} public function dot(string $dot): self { $this->dot = $dot; + $this->pointer = DotsConverter::toPointer($dot); return $this; } @@ -117,7 +124,7 @@ public function pageName(string $name): self } /** - * Set the number of the first page + * Set the number of the first page. */ public function firstPage(int $page): self { @@ -127,17 +134,18 @@ public function firstPage(int $page): self } /** - * Set the total number of pages + * Set the total number of pages. */ public function totalPages(Closure|string $totalPages): self { - $this->totalPages = $this->integerFromResponse($totalPages, minimum: 1); + dd($this->integerFromResponse($totalPages, minimum: 1)); + $this->totalPages = 2;/////////////////$this->integerFromResponse($totalPages, minimum: 1); return $this; } /** - * Retrieve an integer from the response + * Retrieve an integer from the response. */ private function integerFromResponse(Closure|string $key, int $minimum = 0): int { @@ -145,7 +153,7 @@ private function integerFromResponse(Closure|string $key, int $minimum = 0): int } /** - * Retrieve a value from the response + * Retrieve a value from the response. */ private function valueFromResponse(Closure|string $key): mixed { @@ -153,7 +161,7 @@ private function valueFromResponse(Closure|string $key): mixed } /** - * Set the total number of items + * Set the total number of items. */ public function totalItems(Closure|string $totalItems): self { @@ -163,7 +171,7 @@ public function totalItems(Closure|string $totalItems): self } /** - * Set the number of items per page and optionally override it + * Set the number of items per page and optionally override it. */ public function perPage(int $perPage, ?string $key = null, int $firstPageItems = 1): self { @@ -175,7 +183,7 @@ public function perPage(int $perPage, ?string $key = null, int $firstPageItems = } /** - * Set the next page + * Set the next page. */ public function nextPage(Closure|string $key): self { @@ -185,7 +193,7 @@ public function nextPage(Closure|string $key): self } /** - * Set the number of the last page + * Set the number of the last page. */ public function lastPage(Closure|string $key): self { @@ -195,59 +203,45 @@ public function lastPage(Closure|string $key): self } /** - * Fetch pages synchronously - * - * @return self + * Fetch pages synchronously. */ public function sync(): self { - return $this->chunk(1); + return $this->async(1); } /** - * Set the number of pages to fetch per chunk - * - * @param int $size - * @return self + * Set the maximum number of concurrent async HTTP requests. */ - public function chunk(int $size): self + public function async(int $max): self { - $this->chunk = max(1, $size); + $this->async = max(1, $max); return $this; } /** - * Set the maximum number of concurrent async HTTP requests - * - * @param int $max - * @return self + * Set the server connection timeout in seconds. */ - public function concurrency(int $max): self + public function connectionTimeout(float $seconds): self { - $this->concurrency = max(0, $max); + $this->connectionTimeout = max(0, $seconds); return $this; } /** - * Set the timeout in seconds - * - * @param int $seconds - * @return self + * Set an HTTP request timeout in seconds. */ - public function timeout(int $seconds): self + public function requestTimeout(float $seconds): self { - $this->timeout = max(0, $seconds); + $this->requestTimeout = max(0, $seconds); return $this; } /** - * Set the number of attempts to fetch pages - * - * @param int $times - * @return self + * Set the number of attempts to fetch pages. */ public function attempts(int $times): self { @@ -257,22 +251,20 @@ public function attempts(int $times): self } /** - * Set the backoff strategy - * - * @param callable $callback - * @return self + * Set the backoff strategy. */ - public function backoff(callable $callback): self + public function backoff(Closure $callback): self { $this->backoff = $callback; return $this; } - public function make(): Config + public function toConfig(): Config { return new Config( $this->dot, + $this->pointer, $this->pageName, $this->firstPage, $this->totalPages, @@ -283,9 +275,9 @@ public function make(): Config $this->nextPage, $this->nextPageKey, $this->lastPage, - $this->chunk, - $this->concurrency, - $this->timeout, + $this->async, + $this->connectionTimeout, + $this->requestTimeout, $this->attempts, $this->backoff, ); From 687ad02b595ad08973e5318853ee59b45c5e6ef0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 19:23:23 +1000 Subject: [PATCH 017/109] Fetch total pages from the response --- src/Services/Api.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Services/Api.php b/src/Services/Api.php index 86030d5..08d4c4e 100644 --- a/src/Services/Api.php +++ b/src/Services/Api.php @@ -138,8 +138,7 @@ public function firstPage(int $page): self */ public function totalPages(Closure|string $totalPages): self { - dd($this->integerFromResponse($totalPages, minimum: 1)); - $this->totalPages = 2;/////////////////$this->integerFromResponse($totalPages, minimum: 1); + $this->totalPages = $this->integerFromResponse($totalPages, minimum: 1); return $this; } From 1527db487ab404f615ca29bf1b6bd069cc4a6222 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 19:32:22 +1000 Subject: [PATCH 018/109] Update configuration --- src/Dtos/Config.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 5522151..9bc2386 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -10,19 +10,21 @@ final class Config { public function __construct( public readonly string $dot, + public readonly string $pointer, public readonly string $pageName, public readonly int $firstPage, public readonly ?int $totalPages, public readonly ?int $totalItems, public readonly ?int $perPage, public readonly ?string $perPageKey, - public readonly int $perPageOverride, + public readonly ?int $perPageOverride, public readonly ?Closure $nextPage, - public readonly string $nextPageKey, - public readonly int $lastPage, - public readonly int $chunk, - public readonly int $concurrency, - public readonly int $timeout, + public readonly ?string $nextPageKey, + public readonly ?int $lastPage, + public readonly int $async, + public readonly int $connectionTimeout, + public readonly int $requestTimeout, public readonly int $attempts, + public readonly ?Closure $backoff, ) {} } From cfae74dd09fb930a74e8b7521aca6bfe096fcc83 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 19:32:51 +1000 Subject: [PATCH 019/109] Focus on total pages --- src/Paginations/AnyPagination.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 1541e74..cd24518 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -18,11 +18,14 @@ class AnyPagination extends Pagination * @var class-string[] */ protected array $supportedPaginations = [ - CursorPagination::class, - CustomPagination::class, - LengthAwarePagination::class, - LinkHeaderPagination::class, - OffsetPagination::class, + // CursorPagination::class, + // CustomPagination::class, + // LastPageAwarePagination::class, + // LimitPagination::class, + // LinkHeaderPagination::class, + // OffsetPagination::class, + // TotalItemsAwarePagination::class, + TotalPagesAwarePagination::class, ]; /** From 95956660e67fbcf478192181b0ca3ee42f43f71e Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 19:33:21 +1000 Subject: [PATCH 020/109] Fix response initialisation --- src/Sources/Endpoint.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index 76b66f7..a6990b9 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -28,7 +28,7 @@ class Endpoint extends Source /** * The HTTP response value object */ - protected Response $response; + protected ?Response $response = null; /** * Determine whether this class can handle the source. @@ -57,7 +57,10 @@ public function request(): RequestInterface */ public function response(?string $key = null): mixed { - $this->response ??= (new Client())->send($this->request()); + if (!$this->response) { + $response = (new Client())->send($this->request()); + $this->response = new Response($response->getBody()->getContents(), $response->getHeaders()); + } return $key === null ? $this->response : $this->response->get($key); } From af110757b7716a87b9a9d483c0c3302e0e765344 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 19:40:53 +1000 Subject: [PATCH 021/109] Add method to handle a pointer --- src/ValueObjects/Response.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ValueObjects/Response.php b/src/ValueObjects/Response.php index 7d197db..3abfe69 100644 --- a/src/ValueObjects/Response.php +++ b/src/ValueObjects/Response.php @@ -5,6 +5,7 @@ namespace Cerbero\LazyJsonPages\ValueObjects; use Cerbero\JsonParser\JsonParser; +use Cerbero\LazyJson\Pointers\DotsConverter; /** * The HTTP response. @@ -74,8 +75,17 @@ public function header(string $header): ?string */ public function json(string $key): mixed { - $array = JsonParser::parse($this->json)->pointer($key)->toArray(); + $pointer = DotsConverter::toPointer($key); + $array = JsonParser::parse($this->json)->pointer($pointer)->toArray(); return empty($array) ? null : current($array); } + + /** + * Retrieve an iterator with the given JSON pointer. + */ + public function pointer(string $pointer): JsonParser + { + return JsonParser::parse($this->json)->pointer($pointer); + } } From 2d5c29f61c5b2600321cbeedf07ebbfd3fd5a4b4 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 20:54:34 +1000 Subject: [PATCH 022/109] Initialise Pest --- tests/Feature/ExampleTest.php | 5 ++++ tests/Pest.php | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/Feature/ExampleTest.php create mode 100644 tests/Pest.php diff --git a/tests/Feature/ExampleTest.php b/tests/Feature/ExampleTest.php new file mode 100644 index 0000000..b27671c --- /dev/null +++ b/tests/Feature/ExampleTest.php @@ -0,0 +1,5 @@ +expect(true) + ->toBe(true); diff --git a/tests/Pest.php b/tests/Pest.php new file mode 100644 index 0000000..baddc84 --- /dev/null +++ b/tests/Pest.php @@ -0,0 +1,45 @@ +in('Feature'); + +/* +|-------------------------------------------------------------------------- +| Expectations +|-------------------------------------------------------------------------- +| +| When you're writing tests, you often need to check that values meet certain conditions. The +| "expect()" function gives you access to a set of "expectations" methods that you can use +| to assert different things. Of course, you may extend the Expectation API at any time. +| +*/ + +// expect()->extend('toBeOne', function () { +// return $this->toBe(1); +// }); + +/* +|-------------------------------------------------------------------------- +| Functions +|-------------------------------------------------------------------------- +| +| While Pest is very powerful out-of-the-box, you may have some testing code specific to your +| project that you don't want to repeat in every file. Here you can also expose helpers as +| global functions to help you to reduce the number of lines of code in your test files. +| +*/ + +// function something() +// { +// // .. +// } From 30d0dc375c0a3510a99ee8a308b29baf01572c8b Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 20:55:27 +1000 Subject: [PATCH 023/109] Add exception --- src/Exceptions/OutOfAttemptsException.php | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/Exceptions/OutOfAttemptsException.php diff --git a/src/Exceptions/OutOfAttemptsException.php b/src/Exceptions/OutOfAttemptsException.php new file mode 100644 index 0000000..3535e7e --- /dev/null +++ b/src/Exceptions/OutOfAttemptsException.php @@ -0,0 +1,38 @@ + + */ + public readonly LazyCollection $items; + + /** + * Instantiate the class. + */ + public function __construct(TransferException $e, Outcome $outcome) + { + $this->failedPages = $outcome->pullFailedPages(); + $this->items = new LazyCollection(fn() => yield from $outcome->pullItems()); + + parent::__construct($e->getMessage(), 0, $e); + } +} From 63c6866fb5db77de71081391b0459db6490488c5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 20:55:51 +1000 Subject: [PATCH 024/109] Implement fetching outcome --- src/Services/Outcome.php | 78 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/Services/Outcome.php diff --git a/src/Services/Outcome.php b/src/Services/Outcome.php new file mode 100644 index 0000000..9ea4a7f --- /dev/null +++ b/src/Services/Outcome.php @@ -0,0 +1,78 @@ +> + */ + private array $itemsByPage = []; + + /** + * The pages unable to be fetched. + * + * @var int[] + */ + private array $failedPages = []; + + /** + * Add the yielded items from the given page. + * + * @param Traversable $items + */ + public function addItemsFromPage(int $page, Traversable $items): self + { + $this->itemsByPage[$page] = $items; + + return $this; + } + + /** + * Traverse and unset the items. + * + * @return Generator + */ + public function pullItems(): Generator + { + ksort($this->itemsByPage); + + foreach ($this->itemsByPage as $page => $items) { + yield from $items; + + unset($this->itemsByPage[$page]); + } + } + + /** + * Add the given failed page. + */ + public function addFailedPage(int $page): self + { + $this->failedPages[] = $page; + + return $this; + } + + /** + * Retrieve and unset the failed pages. + * + * @return int[] + */ + public function pullFailedPages(): array + { + $failedPages = $this->failedPages; + + $this->failedPages = []; + + return $failedPages; + } +} From 359f3a8e2fae49e13a42f19c567f18b3477ed73d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 20:56:08 +1000 Subject: [PATCH 025/109] Implement concerns --- src/Concerns/PaginationLengthAware.php | 48 ++++++++++++++ src/Concerns/RetriesHttpRequests.php | 85 ++++++++++++++++++++++++ src/Concerns/SendsAsyncRequests.php | 90 ++++++++++++++++++++++++++ 3 files changed, 223 insertions(+) create mode 100644 src/Concerns/PaginationLengthAware.php create mode 100644 src/Concerns/RetriesHttpRequests.php create mode 100644 src/Concerns/SendsAsyncRequests.php diff --git a/src/Concerns/PaginationLengthAware.php b/src/Concerns/PaginationLengthAware.php new file mode 100644 index 0000000..4da4659 --- /dev/null +++ b/src/Concerns/PaginationLengthAware.php @@ -0,0 +1,48 @@ + + */ + protected function itemsByTotalPages(int $pages, ?UriInterface $uri = null): Traversable + { + $uri ??= $this->source->request()->getUri(); + $firstPageAlreadyFetched = strval($uri) == strval($this->source->request()->getUri()); + $chunkedPages = $this->chunkPages($pages, $firstPageAlreadyFetched); + $items = $this->fetchItemsAsynchronously($chunkedPages, $uri); + + if ($firstPageAlreadyFetched) { + yield from $this->source->response()->pointer($this->config->pointer); + } + + yield from $items; + } + + /** + * Retrieve the given pages in chunks. + * + * @return LazyCollection + */ + protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection + { + $firstPage = $shouldSkipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; + $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; + + return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async ?: INF); + } +} diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php new file mode 100644 index 0000000..368aae9 --- /dev/null +++ b/src/Concerns/RetriesHttpRequests.php @@ -0,0 +1,85 @@ +config->attempts; + + do { + $attempt++; + $remainingAttempts--; + + try { + return $callback($outcome); + } catch (TransferException $e) { + if ($remainingAttempts > 0) { + $this->backoff($attempt); + } else { + throw new OutOfAttemptsException($e, $outcome); + } + } + } while ($remainingAttempts > 0); + } + + /** + * Execute the backoff strategy. + */ + protected function backoff(int $attempt): void + { + $backoff = $this->config->backoff ?: fn(int $attempt) => $attempt ** 2 * 100; + + Sleep::for($backoff($attempt) * 1000)->microseconds(); + } + + /** + * Retry to yield HTTP responses from the given callback. + * + * @param callable $callback + * @return Generator + */ + protected function retryYielding(callable $callback): Generator + { + $attempt = 0; + $outcome = new Outcome(); + $remainingAttempts = $this->config->attempts; + + do { + $failed = false; + $attempt++; + $remainingAttempts--; + + try { + yield from $callback($outcome); + } catch (TransferException $e) { + $failed = true; + + if ($remainingAttempts > 0) { + $this->backoff($attempt); + } else { + throw new OutOfAttemptsException($e, $outcome); + } + } + } while ($failed && $remainingAttempts > 0); + } +} diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php new file mode 100644 index 0000000..432b3de --- /dev/null +++ b/src/Concerns/SendsAsyncRequests.php @@ -0,0 +1,90 @@ + $chunkedPages + * @return Traversable + */ + protected function fetchItemsAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable + { + $client = new Client([ + 'timeout' => $this->config->requestTimeout, + 'connect_timeout' => $this->config->connectionTimeout, + ]); + + foreach ($chunkedPages as $pages) { + $outcome = $this->retry(function (Outcome $outcome) use ($uri, $client, $pages) { + $pages = $outcome->pullFailedPages() ?: $pages; + + return $this->pool($client, $outcome, $this->yieldRequests($uri, $pages)); + }); + + yield from $outcome->pullItems(); + } + } + + /** + * Retrieve a generator yielding the HTTP requests for the given pages. + * + * @param int[] $pages + * @return Generator + */ + protected function yieldRequests(UriInterface $uri, array $pages): Generator + { + /** @var RequestInterface $request */ + $request = clone $this->source->request(); + + foreach ($pages as $page) { + $pageUri = Uri::withQueryValue($uri, $this->config->pageName, (string) $page); + + yield $page => $request->withUri($pageUri); + } + } + + /** + * Retrieve the outcome of a pool of asynchronous requests. + * + * @param Generator $requests + */ + protected function pool(Client $client, Outcome $outcome, Generator $requests): Outcome + { + $pool = new Pool($client, $requests, [ + 'concurrency' => $this->config->async, + 'fulfilled' => function (ResponseInterface $response, int $page) use ($outcome) { + $outcome->addItemsFromPage($page, JsonParser::parse($response)->pointer($this->config->pointer)); + }, + 'rejected' => function (Throwable $e, int $page) use ($outcome) { + $outcome->addFailedPage($page); + throw $e; + } + ]); + + $pool->promise()->wait(); + + return $outcome; + } +} From 63900f4788bdbcfb02afb4ee11b530a6bcd7f0de Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 20:56:54 +1000 Subject: [PATCH 026/109] Implement total pages aware pagination --- src/Paginations/TotalPagesAwarePagination.php | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/Paginations/TotalPagesAwarePagination.php diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php new file mode 100644 index 0000000..57c0896 --- /dev/null +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -0,0 +1,35 @@ +config->totalPages !== null + && $this->config->perPage === null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + yield from $this->itemsByTotalPages($this->config->totalPages); + } +} From 8017d7095bfcac20802d3f11abaa0008dc1aafd5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 11 Jan 2024 22:04:34 +1000 Subject: [PATCH 027/109] Lint --- src/Concerns/PaginationLengthAware.php | 4 ++-- src/Concerns/RetriesHttpRequests.php | 5 ++++- src/Concerns/SendsAsyncRequests.php | 10 ++++++---- src/Dtos/Config.php | 4 ++-- src/Exceptions/LazyJsonPagesException.php | 6 +----- src/Services/Api.php | 4 ++-- src/Sources/Endpoint.php | 2 +- src/ValueObjects/Response.php | 18 ++++++++++++------ 8 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/Concerns/PaginationLengthAware.php b/src/Concerns/PaginationLengthAware.php index 4da4659..ea19fab 100644 --- a/src/Concerns/PaginationLengthAware.php +++ b/src/Concerns/PaginationLengthAware.php @@ -36,13 +36,13 @@ protected function itemsByTotalPages(int $pages, ?UriInterface $uri = null): Tra /** * Retrieve the given pages in chunks. * - * @return LazyCollection + * @return LazyCollection> */ protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection { $firstPage = $shouldSkipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; - return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async ?: INF); + return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); } } diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php index 368aae9..8c9ae20 100644 --- a/src/Concerns/RetriesHttpRequests.php +++ b/src/Concerns/RetriesHttpRequests.php @@ -18,7 +18,10 @@ trait RetriesHttpRequests /** * Retry to return HTTP responses from the given callback. * - * @param Closure(Outcome) $callback + * @template TReturn + * + * @param (Closure(Outcome): TReturn) $callback + * @return TReturn */ protected function retry(Closure $callback): mixed { diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 432b3de..86ae466 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -26,7 +26,7 @@ trait SendsAsyncRequests /** * Fetch items by performing asynchronous HTTP calls. * - * @param LazyCollection $chunkedPages + * @param LazyCollection> $chunkedPages * @return Traversable */ protected function fetchItemsAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable @@ -37,8 +37,8 @@ protected function fetchItemsAsynchronously(LazyCollection $chunkedPages, UriInt ]); foreach ($chunkedPages as $pages) { - $outcome = $this->retry(function (Outcome $outcome) use ($uri, $client, $pages) { - $pages = $outcome->pullFailedPages() ?: $pages; + $outcome = $this->retry(function (Outcome $outcome) use ($uri, $client, $pages): Outcome { + $pages = $outcome->pullFailedPages() ?: $pages->all(); return $this->pool($client, $outcome, $this->yieldRequests($uri, $pages)); }); @@ -75,7 +75,9 @@ protected function pool(Client $client, Outcome $outcome, Generator $requests): $pool = new Pool($client, $requests, [ 'concurrency' => $this->config->async, 'fulfilled' => function (ResponseInterface $response, int $page) use ($outcome) { - $outcome->addItemsFromPage($page, JsonParser::parse($response)->pointer($this->config->pointer)); + /** @var Traversable $items */ + $items = JsonParser::parse($response)->pointer($this->config->pointer); + $outcome->addItemsFromPage($page, $items); }, 'rejected' => function (Throwable $e, int $page) use ($outcome) { $outcome->addFailedPage($page); diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 9bc2386..aba8655 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -22,8 +22,8 @@ public function __construct( public readonly ?string $nextPageKey, public readonly ?int $lastPage, public readonly int $async, - public readonly int $connectionTimeout, - public readonly int $requestTimeout, + public readonly float|int $connectionTimeout, + public readonly float|int $requestTimeout, public readonly int $attempts, public readonly ?Closure $backoff, ) {} diff --git a/src/Exceptions/LazyJsonPagesException.php b/src/Exceptions/LazyJsonPagesException.php index 30d56a9..6fe7aa9 100644 --- a/src/Exceptions/LazyJsonPagesException.php +++ b/src/Exceptions/LazyJsonPagesException.php @@ -5,12 +5,8 @@ namespace Cerbero\LazyJsonPages\Exceptions; use Exception; -use Throwable; class LazyJsonPagesException extends Exception { - public static function from(Throwable $e): static - { - return $e instanceof static ? $e : new static($e->getMessage()); - } + // } diff --git a/src/Services/Api.php b/src/Services/Api.php index 08d4c4e..3391de8 100644 --- a/src/Services/Api.php +++ b/src/Services/Api.php @@ -80,12 +80,12 @@ final class Api /** * The server connection timeout in seconds. */ - private int $connectionTimeout = 5; + private float|int $connectionTimeout = 5; /** * The HTTP request timeout in seconds. */ - private int $requestTimeout = 5; + private float|int $requestTimeout = 5; /** * The number of attempts to fetch pages. diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index a6990b9..a0cb3c0 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -36,7 +36,7 @@ class Endpoint extends Source public function matches(): bool { return $this->source instanceof UriInterface - || (is_string($this->source) && $this->isEndpoint($this->source)); + || $this->isEndpoint($this->source); } /** diff --git a/src/ValueObjects/Response.php b/src/ValueObjects/Response.php index 3abfe69..9523604 100644 --- a/src/ValueObjects/Response.php +++ b/src/ValueObjects/Response.php @@ -6,6 +6,7 @@ use Cerbero\JsonParser\JsonParser; use Cerbero\LazyJson\Pointers\DotsConverter; +use Traversable; /** * The HTTP response. @@ -15,14 +16,14 @@ final class Response /** * The headers of the HTTP response. * - * @var array $headers + * @var array $headers */ public readonly array $headers; /** * Instantiate the class. * - * @param array $headers + * @param array $headers */ public function __construct(public readonly string $json, array $headers) { @@ -32,8 +33,8 @@ public function __construct(public readonly string $json, array $headers) /** * Normalize the given headers. * - * @param array $headers - * @return array + * @param array $headers + * @return array */ private function normalizeHeaders(array $headers): array { @@ -64,8 +65,10 @@ public function hasHeader(string $header): bool /** * Retrieve the given header. + * + * @return ?string[] */ - public function header(string $header): ?string + public function header(string $header): ?array { return $this->headers[strtolower($header)] ?? null; } @@ -83,9 +86,12 @@ public function json(string $key): mixed /** * Retrieve an iterator with the given JSON pointer. + * + * @return Traversable */ - public function pointer(string $pointer): JsonParser + public function pointer(string $pointer): Traversable { + /** @var Traversable */ return JsonParser::parse($this->json)->pointer($pointer); } } From 00ca9a34a9ef2daefaf70964e69e20a580144d76 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 15 Jan 2024 17:45:01 +1000 Subject: [PATCH 028/109] Improve DX by moving config hydration to the entry-point --- src/Dtos/Config.php | 34 ++--- src/LazyJsonPages.php | 210 +++++++++++++++++++++++++++---- src/Services/Api.php | 284 ------------------------------------------ 3 files changed, 200 insertions(+), 328 deletions(-) delete mode 100644 src/Services/Api.php diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index aba8655..df9c0b1 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -9,22 +9,22 @@ final class Config { public function __construct( - public readonly string $dot, - public readonly string $pointer, - public readonly string $pageName, - public readonly int $firstPage, - public readonly ?int $totalPages, - public readonly ?int $totalItems, - public readonly ?int $perPage, - public readonly ?string $perPageKey, - public readonly ?int $perPageOverride, - public readonly ?Closure $nextPage, - public readonly ?string $nextPageKey, - public readonly ?int $lastPage, - public readonly int $async, - public readonly float|int $connectionTimeout, - public readonly float|int $requestTimeout, - public readonly int $attempts, - public readonly ?Closure $backoff, + public readonly string $dot = "*", + public readonly string $pointer = '', + public readonly string $pageName = 'page', + public readonly int $firstPage = 1, + public readonly ?int $totalPages = null, + public readonly ?int $totalItems = null, + public readonly ?int $perPage = null, + public readonly ?string $perPageKey = null, + public readonly ?int $perPageOverride = null, + public readonly ?Closure $nextPage = null, + public readonly ?string $nextPageKey = null, + public readonly ?int $lastPage = null, + public readonly int $async = 3, + public readonly float|int $connectionTimeout = 5, + public readonly float|int $requestTimeout = 5, + public readonly int $attempts = 3, + public readonly ?Closure $backoff = null, ) {} } diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 23bb0b6..d01e7dc 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -4,55 +4,211 @@ namespace Cerbero\LazyJsonPages; +use Cerbero\LazyJson\Pointers\DotsConverter; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Paginations\AnyPagination; -use Cerbero\LazyJsonPages\Services\Api; use Cerbero\LazyJsonPages\Sources\AnySource; +use Cerbero\LazyJsonPages\ValueObjects\Response; use Closure; use Illuminate\Support\LazyCollection; -use IteratorAggregate; -use Traversable; /** - * The Lazy JSON Pages entry-point. - * - * @implements IteratorAggregate + * The Lazy JSON Pages entry-point */ -final class LazyJsonPages implements IteratorAggregate +final class LazyJsonPages { /** - * Instantiate the class statically. + * The source of the paginated API. + */ + private readonly AnySource $source; + + /** + * The raw configuration of the API pagination. * - * @param Closure(Api): void $configure - * @return LazyCollection + * @var array */ - public static function from(mixed $source, Closure $configure): LazyCollection - { - $source = new AnySource($source); - $configure($api = new Api($source)); + private array $config = []; - return new LazyCollection(fn() => yield from new self($source, $api->toConfig())); + /** + * Instantiate the class statically. + */ + public static function from(mixed $source): self + { + return new self($source); } /** * Instantiate the class. */ - private function __construct( - private readonly AnySource $source, - private readonly Config $config, - ) {} + public function __construct(mixed $source) + { + $this->source = new AnySource($source); + } + + /** + * Set the name of the page. + */ + public function pageName(string $name): self + { + $this->config['pageName'] = $name; + + return $this; + } + + /** + * Set the number of the first page. + */ + public function firstPage(int $page): self + { + $this->config['firstPage'] = max(0, $page); + + return $this; + } + + /** + * Set the total number of pages. + */ + public function totalPages(Closure|string $totalPages): self + { + $this->config['totalPages'] = $this->integerFromResponse($totalPages, minimum: 1); + + return $this; + } + + /** + * Retrieve an integer from the response. + */ + private function integerFromResponse(Closure|string $key, int $minimum = 0): int + { + return (int) max($minimum, $this->valueFromResponse($key)); + } + + /** + * Retrieve a value from the response. + */ + private function valueFromResponse(Closure|string $key): mixed + { + return $key instanceof Closure ? $key($this->source->response()) : $this->source->response($key); + } + + /** + * Set the total number of items. + */ + public function totalItems(Closure|string $totalItems): self + { + $this->config['totalItems'] = $this->integerFromResponse($totalItems); + + return $this; + } + + /** + * Set the number of items per page and optionally override it. + */ + public function perPage(int $perPage, ?string $key = null, int $firstPageItems = 1): self + { + $this->config['perPage'] = max(1, $key ? $firstPageItems : $perPage); + $this->config['perPageKey'] = $key; + $this->config['perPageOverride'] = $key ? max(1, $perPage) : null; + + return $this; + } + + /** + * Set the next page. + */ + public function nextPage(Closure|string $key): self + { + $this->config['nextPage'] = $key instanceof Closure ? $key : fn(Response $response) => $response->get($key); + + return $this; + } + + /** + * Set the number of the last page. + */ + public function lastPage(Closure|string $key): self + { + $this->config['lastPage'] = $this->integerFromResponse($key); + + return $this; + } + + /** + * Fetch pages synchronously. + */ + public function sync(): self + { + return $this->async(1); + } /** - * Retrieve the paginated items lazily. + * Set the maximum number of concurrent async HTTP requests. + */ + public function async(int $max): self + { + $this->config['async'] = max(1, $max); + + return $this; + } + + /** + * Set the server connection timeout in seconds. + */ + public function connectionTimeout(float $seconds): self + { + $this->config['connectionTimeout'] = max(0, $seconds); + + return $this; + } + + /** + * Set an HTTP request timeout in seconds. + */ + public function requestTimeout(float $seconds): self + { + $this->config['requestTimeout'] = max(0, $seconds); + + return $this; + } + + /** + * Set the number of attempts to fetch pages. + */ + public function attempts(int $times): self + { + $this->config['attempts'] = max(1, $times); + + return $this; + } + + /** + * Set the backoff strategy. + */ + public function backoff(Closure $callback): self + { + $this->config['backoff'] = $callback; + + return $this; + } + + /** + * Retrieve a lazy collection yielding the paginated items. * - * @return Traversable + * @return LazyCollection */ - public function getIterator(): Traversable + public function collect(string $dot = '*'): LazyCollection { - // yield each item within a loop - instead of using `yield from` - to ignore the actual item index - // and ensure indexes continuity, otherwise the index of items always starts from 0 on every page. - foreach (new AnyPagination($this->source, $this->config) as $item) { - yield $item; - } + $this->config['dot'] = $dot; + $this->config['pointer'] = DotsConverter::toPointer($dot); + + return new LazyCollection(function () { + $items = new AnyPagination($this->source, new Config(...$this->config)); + + // yield each item within a loop - instead of using `yield from` - to ignore the actual item index + // and ensure indexes continuity, otherwise the index of items always starts from 0 on every page. + foreach ($items as $item) { + yield $item; + } + }); } } diff --git a/src/Services/Api.php b/src/Services/Api.php deleted file mode 100644 index 3391de8..0000000 --- a/src/Services/Api.php +++ /dev/null @@ -1,284 +0,0 @@ -dot = $dot; - $this->pointer = DotsConverter::toPointer($dot); - - return $this; - } - - /** - * Set the name of the page. - */ - public function pageName(string $name): self - { - $this->pageName = $name; - - return $this; - } - - /** - * Set the number of the first page. - */ - public function firstPage(int $page): self - { - $this->firstPage = max(0, $page); - - return $this; - } - - /** - * Set the total number of pages. - */ - public function totalPages(Closure|string $totalPages): self - { - $this->totalPages = $this->integerFromResponse($totalPages, minimum: 1); - - return $this; - } - - /** - * Retrieve an integer from the response. - */ - private function integerFromResponse(Closure|string $key, int $minimum = 0): int - { - return (int) max($minimum, $this->valueFromResponse($key)); - } - - /** - * Retrieve a value from the response. - */ - private function valueFromResponse(Closure|string $key): mixed - { - return $key instanceof Closure ? $key($this->source->response()) : $this->source->response($key); - } - - /** - * Set the total number of items. - */ - public function totalItems(Closure|string $totalItems): self - { - $this->totalItems = $this->integerFromResponse($totalItems); - - return $this; - } - - /** - * Set the number of items per page and optionally override it. - */ - public function perPage(int $perPage, ?string $key = null, int $firstPageItems = 1): self - { - $this->perPage = max(1, $key ? $firstPageItems : $perPage); - $this->perPageKey = $key; - $this->perPageOverride = $key ? max(1, $perPage) : null; - - return $this; - } - - /** - * Set the next page. - */ - public function nextPage(Closure|string $key): self - { - $this->nextPage = $key instanceof Closure ? $key : fn(Response $response) => $response->get($key); - - return $this; - } - - /** - * Set the number of the last page. - */ - public function lastPage(Closure|string $key): self - { - $this->lastPage = $this->integerFromResponse($key); - - return $this; - } - - /** - * Fetch pages synchronously. - */ - public function sync(): self - { - return $this->async(1); - } - - /** - * Set the maximum number of concurrent async HTTP requests. - */ - public function async(int $max): self - { - $this->async = max(1, $max); - - return $this; - } - - /** - * Set the server connection timeout in seconds. - */ - public function connectionTimeout(float $seconds): self - { - $this->connectionTimeout = max(0, $seconds); - - return $this; - } - - /** - * Set an HTTP request timeout in seconds. - */ - public function requestTimeout(float $seconds): self - { - $this->requestTimeout = max(0, $seconds); - - return $this; - } - - /** - * Set the number of attempts to fetch pages. - */ - public function attempts(int $times): self - { - $this->attempts = max(1, $times); - - return $this; - } - - /** - * Set the backoff strategy. - */ - public function backoff(Closure $callback): self - { - $this->backoff = $callback; - - return $this; - } - - public function toConfig(): Config - { - return new Config( - $this->dot, - $this->pointer, - $this->pageName, - $this->firstPage, - $this->totalPages, - $this->totalItems, - $this->perPage, - $this->perPageKey, - $this->perPageOverride, - $this->nextPage, - $this->nextPageKey, - $this->lastPage, - $this->async, - $this->connectionTimeout, - $this->requestTimeout, - $this->attempts, - $this->backoff, - ); - } -} From 88bfd80ed5e1eb938f1cabeb75a66af239cd7a7c Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 15 Jan 2024 17:48:20 +1000 Subject: [PATCH 029/109] Adjust request options --- src/Concerns/SendsAsyncRequests.php | 6 ++++-- src/Sources/Endpoint.php | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 86ae466..062f9e0 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -8,6 +8,7 @@ use GuzzleHttp\Client; use GuzzleHttp\Pool; use GuzzleHttp\Psr7\Uri; +use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -32,8 +33,9 @@ trait SendsAsyncRequests protected function fetchItemsAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable { $client = new Client([ - 'timeout' => $this->config->requestTimeout, - 'connect_timeout' => $this->config->connectionTimeout, + RequestOptions::TIMEOUT => $this->config->requestTimeout, + RequestOptions::CONNECT_TIMEOUT => $this->config->connectionTimeout, + RequestOptions::STREAM => true, ]); foreach ($chunkedPages as $pages) { diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index a0cb3c0..e6cd8a2 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -8,6 +8,7 @@ use Cerbero\LazyJsonPages\ValueObjects\Response; use GuzzleHttp\Client; use GuzzleHttp\Psr7\Request; +use GuzzleHttp\RequestOptions; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\UriInterface; @@ -58,7 +59,7 @@ public function request(): RequestInterface public function response(?string $key = null): mixed { if (!$this->response) { - $response = (new Client())->send($this->request()); + $response = (new Client([RequestOptions::STREAM => true]))->send($this->request()); $this->response = new Response($response->getBody()->getContents(), $response->getHeaders()); } From 5b5e4e6b17999bb3f9d60863a2f062d0682da955 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 15 Jan 2024 17:48:52 +1000 Subject: [PATCH 030/109] Rely on abstraction --- src/Paginations/Pagination.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index b66424a..b785968 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -5,7 +5,7 @@ namespace Cerbero\LazyJsonPages\Paginations; use Cerbero\LazyJsonPages\Dtos\Config; -use Cerbero\LazyJsonPages\Sources\AnySource; +use Cerbero\LazyJsonPages\Sources\Source; use IteratorAggregate; use Traversable; @@ -29,7 +29,7 @@ abstract public function matches(): bool; abstract public function getIterator(): Traversable; final public function __construct( - protected readonly AnySource $source, + protected readonly Source $source, protected readonly Config $config, ) {} } From 06ad83ed2ac98a274690b857b12df834a9d2757a Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:31:46 +1000 Subject: [PATCH 031/109] Remove bootstrap --- bootstrap.php | 10 ---------- composer.json | 1 - 2 files changed, 11 deletions(-) delete mode 100644 bootstrap.php diff --git a/bootstrap.php b/bootstrap.php deleted file mode 100644 index c658bd3..0000000 --- a/bootstrap.php +++ /dev/null @@ -1,10 +0,0 @@ - Date: Sat, 20 Jan 2024 20:34:04 +1000 Subject: [PATCH 032/109] Improve configuration --- src/Dtos/Config.php | 4 +--- src/LazyJsonPages.php | 38 +++++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index df9c0b1..f09e3a6 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -13,7 +13,7 @@ public function __construct( public readonly string $pointer = '', public readonly string $pageName = 'page', public readonly int $firstPage = 1, - public readonly ?int $totalPages = null, + public readonly ?string $totalPagesKey = null, public readonly ?int $totalItems = null, public readonly ?int $perPage = null, public readonly ?string $perPageKey = null, @@ -22,8 +22,6 @@ public function __construct( public readonly ?string $nextPageKey = null, public readonly ?int $lastPage = null, public readonly int $async = 3, - public readonly float|int $connectionTimeout = 5, - public readonly float|int $requestTimeout = 5, public readonly int $attempts = 3, public readonly ?Closure $backoff = null, ) {} diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index d01e7dc..f47d793 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -7,9 +7,11 @@ use Cerbero\LazyJson\Pointers\DotsConverter; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Paginations\AnyPagination; +use Cerbero\LazyJsonPages\Services\Client; use Cerbero\LazyJsonPages\Sources\AnySource; use Cerbero\LazyJsonPages\ValueObjects\Response; use Closure; +use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; /** @@ -29,6 +31,15 @@ final class LazyJsonPages */ private array $config = []; + /** + * The Guzzle HTTP request options. + */ + private array $requestOptions = [ + RequestOptions::CONNECT_TIMEOUT => 5, + RequestOptions::READ_TIMEOUT => 5, + RequestOptions::TIMEOUT => 5, + ]; + /** * Instantiate the class statically. */ @@ -68,9 +79,9 @@ public function firstPage(int $page): self /** * Set the total number of pages. */ - public function totalPages(Closure|string $totalPages): self + public function totalPages(string $key): self { - $this->config['totalPages'] = $this->integerFromResponse($totalPages, minimum: 1); + $this->config['totalPagesKey'] = $key; return $this; } @@ -104,11 +115,11 @@ public function totalItems(Closure|string $totalItems): self /** * Set the number of items per page and optionally override it. */ - public function perPage(int $perPage, ?string $key = null, int $firstPageItems = 1): self + public function perPage(int $items, ?string $key = null, int $firstPageItems = 1): self { - $this->config['perPage'] = max(1, $key ? $firstPageItems : $perPage); + $this->config['perPage'] = max(1, $key ? $firstPageItems : $items); $this->config['perPageKey'] = $key; - $this->config['perPageOverride'] = $key ? max(1, $perPage) : null; + $this->config['perPageOverride'] = $key ? max(1, $items) : null; return $this; } @@ -144,9 +155,9 @@ public function sync(): self /** * Set the maximum number of concurrent async HTTP requests. */ - public function async(int $max): self + public function async(int $requests): self { - $this->config['async'] = max(1, $max); + $this->config['async'] = max(1, $requests); return $this; } @@ -154,9 +165,9 @@ public function async(int $max): self /** * Set the server connection timeout in seconds. */ - public function connectionTimeout(float $seconds): self + public function connectionTimeout(float|int $seconds): self { - $this->config['connectionTimeout'] = max(0, $seconds); + $this->requestOptions[RequestOptions::CONNECT_TIMEOUT] = max(0, $seconds); return $this; } @@ -164,9 +175,10 @@ public function connectionTimeout(float $seconds): self /** * Set an HTTP request timeout in seconds. */ - public function requestTimeout(float $seconds): self + public function requestTimeout(float|int $seconds): self { - $this->config['requestTimeout'] = max(0, $seconds); + $this->requestOptions[RequestOptions::TIMEOUT] = max(0, $seconds); + $this->requestOptions[RequestOptions::READ_TIMEOUT] = max(0, $seconds); return $this; } @@ -201,6 +213,8 @@ public function collect(string $dot = '*'): LazyCollection $this->config['dot'] = $dot; $this->config['pointer'] = DotsConverter::toPointer($dot); + Client::configure($this->requestOptions); + return new LazyCollection(function () { $items = new AnyPagination($this->source, new Config(...$this->config)); @@ -209,6 +223,8 @@ public function collect(string $dot = '*'): LazyCollection foreach ($items as $item) { yield $item; } + + Client::reset(); }); } } From 99356dc4c4df3a1f8cca1a60d39476339dc53023 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:36:25 +1000 Subject: [PATCH 033/109] Move length-aware logic to proper pagination --- src/Concerns/PaginationLengthAware.php | 48 ------------ src/Paginations/LengthAwarePagination.php | 74 +++++++++++++++++++ src/Paginations/TotalPagesAwarePagination.php | 11 +-- 3 files changed, 78 insertions(+), 55 deletions(-) delete mode 100644 src/Concerns/PaginationLengthAware.php create mode 100644 src/Paginations/LengthAwarePagination.php diff --git a/src/Concerns/PaginationLengthAware.php b/src/Concerns/PaginationLengthAware.php deleted file mode 100644 index ea19fab..0000000 --- a/src/Concerns/PaginationLengthAware.php +++ /dev/null @@ -1,48 +0,0 @@ - - */ - protected function itemsByTotalPages(int $pages, ?UriInterface $uri = null): Traversable - { - $uri ??= $this->source->request()->getUri(); - $firstPageAlreadyFetched = strval($uri) == strval($this->source->request()->getUri()); - $chunkedPages = $this->chunkPages($pages, $firstPageAlreadyFetched); - $items = $this->fetchItemsAsynchronously($chunkedPages, $uri); - - if ($firstPageAlreadyFetched) { - yield from $this->source->response()->pointer($this->config->pointer); - } - - yield from $items; - } - - /** - * Retrieve the given pages in chunks. - * - * @return LazyCollection> - */ - protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection - { - $firstPage = $shouldSkipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; - $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; - - return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); - } -} diff --git a/src/Paginations/LengthAwarePagination.php b/src/Paginations/LengthAwarePagination.php new file mode 100644 index 0000000..025f2a1 --- /dev/null +++ b/src/Paginations/LengthAwarePagination.php @@ -0,0 +1,74 @@ + + */ + protected function yieldItemsUntilPage(string $key, ?Closure $callback = null): Generator + { + yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $key); + + $page = $this->toPage($generator->getReturn()); + + if (!is_int($page)) { + throw new InvalidPageException($key); + } + + $page = $callback ? $callback($page) : $page; + + foreach ($this->yieldPageResponsesUntil($page) as $pageResponse) { + yield from $this->yieldItemsFrom($pageResponse); + } + } + + /** + * Yield the HTTP page responses until given page. + * + * @return Generator + */ + protected function yieldPageResponsesUntil(int $page, ?UriInterface $uri = null): Generator + { + $uri ??= $this->source->request()->getUri(); + $firstPageAlreadyFetched = strval($uri) == strval($this->source->request()->getUri()); + $chunkedPages = $this->chunkPages($page, $firstPageAlreadyFetched); + + yield from $this->fetchPagesAsynchronously($chunkedPages, $uri); + } + + /** + * Retrieve the given pages in chunks. + * + * @return LazyCollection> + */ + protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection + { + if ($pages == 1 && $shouldSkipFirstPage) { + return LazyCollection::empty(); + } + + $firstPage = $shouldSkipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; + $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; + + return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); + } +} diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index 57c0896..462f662 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -4,32 +4,29 @@ namespace Cerbero\LazyJsonPages\Paginations; -use Cerbero\LazyJsonPages\Concerns\PaginationLengthAware; use Traversable; /** * The pagination aware of the total number of pages. */ -class TotalPagesAwarePagination extends Pagination +class TotalPagesAwarePagination extends LengthAwarePagination { - use PaginationLengthAware; - /** * Determine whether the configuration matches this pagination. */ public function matches(): bool { - return $this->config->totalPages !== null + return $this->config->totalPagesKey !== null && $this->config->perPage === null; } /** * Yield the paginated items. * - * @return Traversable + * @return Traversable */ public function getIterator(): Traversable { - yield from $this->itemsByTotalPages($this->config->totalPages); + yield from $this->yieldItemsUntilPage($this->config->totalPagesKey); } } From db5187557ab7285d061be8d1874236ff04ac870b Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:39:47 +1000 Subject: [PATCH 034/109] Replace outcome with book --- src/Concerns/RetriesHttpRequests.php | 23 ++++++--- src/Concerns/SendsAsyncRequests.php | 67 +++++++++----------------- src/Services/{Outcome.php => Book.php} | 36 +++++++------- 3 files changed, 55 insertions(+), 71 deletions(-) rename src/Services/{Outcome.php => Book.php} (50%) diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php index 8c9ae20..fc73b5a 100644 --- a/src/Concerns/RetriesHttpRequests.php +++ b/src/Concerns/RetriesHttpRequests.php @@ -3,7 +3,6 @@ namespace Cerbero\LazyJsonPages\Concerns; use Cerbero\LazyJsonPages\Exceptions\OutOfAttemptsException; -use Cerbero\LazyJsonPages\Services\Outcome; use Closure; use Generator; use GuzzleHttp\Exception\TransferException; @@ -26,7 +25,6 @@ trait RetriesHttpRequests protected function retry(Closure $callback): mixed { $attempt = 0; - $outcome = new Outcome(); $remainingAttempts = $this->config->attempts; do { @@ -34,12 +32,12 @@ protected function retry(Closure $callback): mixed $remainingAttempts--; try { - return $callback($outcome); + return $callback(); } catch (TransferException $e) { if ($remainingAttempts > 0) { $this->backoff($attempt); } else { - throw new OutOfAttemptsException($e, $outcome); + $this->outOfAttempts($e); } } } while ($remainingAttempts > 0); @@ -55,6 +53,18 @@ protected function backoff(int $attempt): void Sleep::for($backoff($attempt) * 1000)->microseconds(); } + /** + * Throw the out of attempts exception. + */ + protected function outOfAttempts(TransferException $e): never + { + throw new OutOfAttemptsException($e, $this->book->pullFailedPages(), function () { + foreach ($this->book->pullPages() as $page) { + yield from $this->yieldItemsFrom($page); + } + }); + } + /** * Retry to yield HTTP responses from the given callback. * @@ -64,7 +74,6 @@ protected function backoff(int $attempt): void protected function retryYielding(callable $callback): Generator { $attempt = 0; - $outcome = new Outcome(); $remainingAttempts = $this->config->attempts; do { @@ -73,14 +82,14 @@ protected function retryYielding(callable $callback): Generator $remainingAttempts--; try { - yield from $callback($outcome); + yield from $callback(); } catch (TransferException $e) { $failed = true; if ($remainingAttempts > 0) { $this->backoff($attempt); } else { - throw new OutOfAttemptsException($e, $outcome); + $this->outOfAttempts($e); } } } while ($failed && $remainingAttempts > 0); diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 062f9e0..7b4df3b 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -2,13 +2,10 @@ namespace Cerbero\LazyJsonPages\Concerns; -use Cerbero\JsonParser\JsonParser; -use Cerbero\LazyJsonPages\Services\Outcome; +use Cerbero\LazyJsonPages\Services\Client; use Generator; -use GuzzleHttp\Client; use GuzzleHttp\Pool; use GuzzleHttp\Psr7\Uri; -use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -25,40 +22,45 @@ trait SendsAsyncRequests use RetriesHttpRequests; /** - * Fetch items by performing asynchronous HTTP calls. + * Fetch pages by performing asynchronous HTTP calls. * * @param LazyCollection> $chunkedPages - * @return Traversable + * @return Traversable */ - protected function fetchItemsAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable + protected function fetchPagesAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable { - $client = new Client([ - RequestOptions::TIMEOUT => $this->config->requestTimeout, - RequestOptions::CONNECT_TIMEOUT => $this->config->connectionTimeout, - RequestOptions::STREAM => true, - ]); - foreach ($chunkedPages as $pages) { - $outcome = $this->retry(function (Outcome $outcome) use ($uri, $client, $pages): Outcome { - $pages = $outcome->pullFailedPages() ?: $pages->all(); - - return $this->pool($client, $outcome, $this->yieldRequests($uri, $pages)); - }); + $this->retry(fn() => $this->pool($uri, $pages->all())->promise()->wait()); - yield from $outcome->pullItems(); + yield from $this->book->pullPages(); } } + /** + * Retrieve a pool of asynchronous requests. + * + * @param array $pages + */ + protected function pool(UriInterface $uri, array $pages): Pool + { + return new Pool(Client::instance(), $this->yieldRequests($uri, $pages), [ + 'concurrency' => $this->config->async, + 'fulfilled' => fn(ResponseInterface $response, int $page) => $this->book->addPage($page, $response), + 'rejected' => fn(Throwable $e, int $page) => $this->book->addFailedPage($page) && throw $e, + ]); + } + /** * Retrieve a generator yielding the HTTP requests for the given pages. * - * @param int[] $pages + * @param array $pages * @return Generator */ protected function yieldRequests(UriInterface $uri, array $pages): Generator { /** @var RequestInterface $request */ $request = clone $this->source->request(); + $pages = $this->book->pullFailedPages() ?: $pages; foreach ($pages as $page) { $pageUri = Uri::withQueryValue($uri, $this->config->pageName, (string) $page); @@ -66,29 +68,4 @@ protected function yieldRequests(UriInterface $uri, array $pages): Generator yield $page => $request->withUri($pageUri); } } - - /** - * Retrieve the outcome of a pool of asynchronous requests. - * - * @param Generator $requests - */ - protected function pool(Client $client, Outcome $outcome, Generator $requests): Outcome - { - $pool = new Pool($client, $requests, [ - 'concurrency' => $this->config->async, - 'fulfilled' => function (ResponseInterface $response, int $page) use ($outcome) { - /** @var Traversable $items */ - $items = JsonParser::parse($response)->pointer($this->config->pointer); - $outcome->addItemsFromPage($page, $items); - }, - 'rejected' => function (Throwable $e, int $page) use ($outcome) { - $outcome->addFailedPage($page); - throw $e; - } - ]); - - $pool->promise()->wait(); - - return $outcome; - } } diff --git a/src/Services/Outcome.php b/src/Services/Book.php similarity index 50% rename from src/Services/Outcome.php rename to src/Services/Book.php index 9ea4a7f..a956443 100644 --- a/src/Services/Outcome.php +++ b/src/Services/Book.php @@ -3,52 +3,50 @@ namespace Cerbero\LazyJsonPages\Services; use Generator; -use Traversable; +use Psr\Http\Message\ResponseInterface; /** - * The outcome of fetching paginated items. + * The collector of pages. */ -final class Outcome +final class Book { /** - * The iterators yielding items from pages. + * The HTTP responses of the fetched pages. * - * @var array> + * @var array */ - private array $itemsByPage = []; + private array $pages = []; /** * The pages unable to be fetched. * - * @var int[] + * @var array */ private array $failedPages = []; /** - * Add the yielded items from the given page. - * - * @param Traversable $items + * Add the HTTP response of the given page. */ - public function addItemsFromPage(int $page, Traversable $items): self + public function addPage(int $page, ResponseInterface $response): self { - $this->itemsByPage[$page] = $items; + $this->pages[$page] = $response; return $this; } /** - * Traverse and unset the items. + * Yield and forget each page. * * @return Generator */ - public function pullItems(): Generator + public function pullPages(): Generator { - ksort($this->itemsByPage); + ksort($this->pages); - foreach ($this->itemsByPage as $page => $items) { - yield from $items; + foreach ($this->pages as $page => $response) { + yield $response; - unset($this->itemsByPage[$page]); + unset($this->pages[$page]); } } @@ -65,7 +63,7 @@ public function addFailedPage(int $page): self /** * Retrieve and unset the failed pages. * - * @return int[] + * @return array */ public function pullFailedPages(): array { From 23bfa8b4a1e2c90bf129f020baaeef3bdd97358a Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:40:52 +1000 Subject: [PATCH 035/109] Let sources return PSR-7 responses --- src/Sources/AnySource.php | 9 +++++---- src/Sources/Endpoint.php | 27 +++++++++------------------ src/Sources/Source.php | 7 ++++--- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index 21d6156..3abfcf1 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -6,6 +6,7 @@ use Cerbero\LazyJsonPages\Exceptions\UnsupportedSourceException; use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; /** * The aggregator of sources. @@ -69,12 +70,12 @@ protected function matchingSource(): Source } /** - * Retrieve the HTTP response or part of it. + * Retrieve the HTTP response. * - * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) + * @return ResponseInterface */ - public function response(?string $key = null): mixed + public function response(): ResponseInterface { - return $this->matchingSource()->response($key); + return $this->matchingSource()->response(); } } diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index e6cd8a2..e328a57 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -5,11 +5,10 @@ namespace Cerbero\LazyJsonPages\Sources; use Cerbero\JsonParser\Concerns\DetectsEndpoints; -use Cerbero\LazyJsonPages\ValueObjects\Response; -use GuzzleHttp\Client; +use Cerbero\LazyJsonPages\Services\Client; use GuzzleHttp\Psr7\Request; -use GuzzleHttp\RequestOptions; use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; /** @@ -24,12 +23,12 @@ class Endpoint extends Source /** * The HTTP request. */ - protected RequestInterface $request; + protected readonly RequestInterface $request; /** * The HTTP response value object */ - protected ?Response $response = null; + protected readonly ResponseInterface $response; /** * Determine whether this class can handle the source. @@ -45,24 +44,16 @@ public function matches(): bool */ public function request(): RequestInterface { - return $this->request ??= new Request('GET', $this->source, [ - 'Accept' => 'application/json', - 'Content-Type' => 'application/json', - ]); + return $this->request ??= new Request('GET', $this->source); } /** - * Retrieve the HTTP response or part of it. + * Retrieve the HTTP response. * - * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) + * @return ResponseInterface */ - public function response(?string $key = null): mixed + public function response(): ResponseInterface { - if (!$this->response) { - $response = (new Client([RequestOptions::STREAM => true]))->send($this->request()); - $this->response = new Response($response->getBody()->getContents(), $response->getHeaders()); - } - - return $key === null ? $this->response : $this->response->get($key); + return $this->response ??= Client::instance()->send($this->request()); } } diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 34dbcaa..6ad9fae 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -5,6 +5,7 @@ namespace Cerbero\LazyJsonPages\Sources; use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; /** * The abstract implementation of a source. @@ -26,9 +27,9 @@ abstract public function matches(): bool; abstract public function request(): RequestInterface; /** - * Retrieve the HTTP response or part of it. + * Retrieve the HTTP response. * - * @return ($key is string ? mixed : \Cerbero\LazyJsonPages\ValueObjects\Response) + * @return ResponseInterface */ - abstract public function response(?string $key = null): mixed; + abstract public function response(): ResponseInterface; } From 483a50310340dab90b41e381fc0f52f10d7affd2 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:41:02 +1000 Subject: [PATCH 036/109] Update readme --- README.md | 301 ++++++++++++++++++++---------------------------------- 1 file changed, 110 insertions(+), 191 deletions(-) diff --git a/README.md b/README.md index 21a7dbc..36df8bd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ๐Ÿผ Lazy JSON Pages +# ๐Ÿ“œ Lazy JSON Pages [![Author][ico-author]][link-author] [![PHP Version][ico-php]][link-php] @@ -12,20 +12,17 @@ [![Total Downloads][ico-downloads]][link-downloads] ```php -$lazyCollection = LazyCollection::fromJsonPages($source, fn (Config $config) => $config - ->dot('data.results') - ->pages('total_pages') - ->perPage(500, 'page_size') - ->chunk(3) - ->timeout(15) - ->attempts(5) - ->backoff(fn (int $attempt) => $attempt ** 2 * 100)); +$lazyCollection = LazyJsonPages::from($source) + ->totalPages('pagination.total_pages') + ->async(requests: 5) + ->collect('data.*'); ``` Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. Need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. + ## ๐Ÿ“ฆ Install Via Composer: @@ -34,266 +31,188 @@ Via Composer: composer require cerbero/lazy-json-pages ``` -## ๐Ÿ”ฎ Usage -- [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) -- [โ†ช๏ธ Cursor and next-page paginations](#-cursor-and-next-page-paginations) -- [๐Ÿ›  Requests fine-tuning](#-requests-fine-tuning) -- [๐Ÿ’ข Errors handling](#-errors-handling) +## ๐Ÿ”ฎ Usage -Loading paginated items of JSON APIs into a lazy collection is possible by calling the collection itself or the included helper: +* [๐Ÿ‘ฃ Basics](#-basics) +* [๐Ÿ’ง Sources](#-sources) +* [๐Ÿ›๏ธ Pagination structure](#-pagination-structure) +* [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) +* [โ†ช๏ธ Cursor and next-page paginations](#-cursor-and-next-page-paginations) +* [๐Ÿš€ Requests optimization](#-requests-optimization) +* [๐Ÿ’ข Errors handling](#-errors-handling) -```php -$items = LazyCollection::fromJsonPages($source, $path, $config); -$items = lazyJsonPages($source, $path, $config); -``` +### ๐Ÿ‘ฃ Basics -The source which paginated items are fetched from can be either a PSR-7 request or a Laravel HTTP client response: +Depending on our coding style, we can call Lazy JSON Pages in 3 different ways: ```php -// the Guzzle request is just an example, any PSR-7 request can be used as well -$source = new GuzzleHttp\Psr7\Request('GET', 'https://paginated-json-api.test'); +use Cerbero\LazyJsonPages\LazyJsonPages; -// Lazy JSON Pages integrates well with Laravel and supports its HTTP client responses -$source = Http::get('https://paginated-json-api.test'); -``` +use function Cerbero\LazyJsonPages\lazyJsonPages; -Lazy JSON Pages only changes the page query parameter when fetching pages. This means that if the first request was authenticated (e.g. via bearer token), the requests to fetch the other pages will be authenticated as well. - -The second argument, `$path`, is the key within JSON APIs holding the paginated items. The path supports dot-notation so if the key is nested, we can define its nesting levels with dots. For example, given the following JSON: - -```json -{ - "data": { - "results": [ - { - "id": 1 - }, - { - "id": 2 - } - ] - } -} -``` +// classic instantiation +$lazyJsonPages = new LazyJsonPages($source); + +// static method (easier methods chaining) +$lazyJsonPages = LazyJsonPages::from($source); -the path to the paginated items would be `data.results`. All nested JSON keys can be defined with dot-notation, including the keys to set in the configuration. +// namespaced helper +$lazyJsonPages = lazyJsonPages($source); +``` -APIs are all different so Lazy JSON Pages allows us to define tailored configurations for each of them. The configuration can be set with the following variants: +The variable `$source` in our examples represents any [source](#-sources) that points to a paginated JSON API. Once we define the source, we can then chain methods to define how the API is paginated: ```php -// assume that the integer indicates the number of pages -// to be used when the number is known (e.g. via previous HTTP request) -lazyJsonPages($source, $path, 10); - -// assume that the string indicates the JSON key holding the number of pages -lazyJsonPages($source, $path, 'total_pages'); - -// set the config with an associative array -// both snake_case and camelCase keys are allowed -lazyJsonPages($source, $path, [ - 'items' => 'total_items', - 'per_page' => 50, -]); - -// set the config through its fluent methods -use Cerbero\LazyJsonPages\Config; - -lazyJsonPages($source, $path, function (Config $config) { - $config->items('total_items')->perPage(50); -}); +$lazyCollection = LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(20) + ->offset() + ->collect('results.*'); ``` -The configuration depends on the type of pagination. Various paginations are supported, including length-aware and cursor paginations. +When calling `collect()`, we indicate that the pagination structure is defined and that we are ready to collect the paginated items within a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections), where we can loop through the items one by one and apply filters and transformations in a memory-efficient way. -### ๐Ÿ“ Length-aware paginations +### ๐Ÿ’ง Sources -The term "length-aware" indicates all paginations that show at least one of the following numbers: -- the total number of pages -- the total number of items -- the number of the last page +A source is any mean that can point to a paginated JSON API. A number of sources are supported by default: + +- **endpoint URIs**, e.g. `https://example.com/api/v1/users` or any instance of `Psr\Http\Message\UriInterface` +- **PSR-7 requests**, i.e. any instance of `Psr\Http\Message\RequestInterface` +- **Laravel HTTP client requests**, i.e. any instance of `Illuminate\Http\Client\Request` +- **Laravel HTTP client responses**, i.e. any instance of `Illuminate\Http\Client\Response` +- **user-defined sources**, i.e. any instance of `Cerbero\LazyJsonPages\Sources\Source` -Lazy JSON Pages only needs one of these numbers to work properly. When setting the number of items, we can also define the number of items shown per page (if we know it) to save some more memory. The following are all valid configurations: +Here are some examples of sources: ```php -// configure the total number of pages: -$config = 10; -$config = 'total_pages'; -$config = ['pages' => 'total_pages']; -$config->pages('total_pages'); - -// configure the total number of items: -$config = ['items' => 500]; -$config = ['items' => 'total_items']; -$config = ['items' => 'total_items', 'per_page' => 50]; -$config->items('total_items'); -$config->items('total_items')->perPage(50); - -// configure the number of the last page: -$config = ['last_page' => 10]; -$config = ['last_page' => 'last_page_key']; -$config = ['last_page' => 'https://paginated-json-api.test?page=10']; -$config->lastPage(10); -$config->lastPage('last_page_key'); -$config->lastPage('https://paginated-json-api.test?page=10'); -``` +// any PSR-7 compatible request is supported, including Guzzle requests +$source = new GuzzleHttp\Psr7\Request('GET', 'https://example.com/api'); -Depending on the APIs, the last page may be indicated as a number or as a URL, Lazy JSON Pages supports both. +// while being framework-agnostic, Lazy JSON Pages integrates well with Laravel +$source = Http::withToken($bearer)->get('https://example.com/api'); +``` -By default this package assumes that the name of the page query parameter is `page` and that the first page is `1`. If that is not the case, we can update the defaults by adding this configuration: -```php -$config->pageName('page_number')->firstPage(0); -// or -$config = [ - 'page_name' => 'page_number', - 'first_page' => 0, -]; -``` +### ๐Ÿ›๏ธ Pagination structure -When dealing with a lot of data, it's a good idea to fetch only 1 item (or a few if 1 is not allowed) on the first page to count the total number of pages/items without wasting memory and then fetch all the calculated pages with many more items. +After defining the [source](#-sources), we need to let Lazy JSON Pages know what the paginated API looks like. -We can do that with the "per page" setting by passing: -- the new number of items to show per page -- the query parameter holding the number of items per page +If the API uses a query parameter different from `page` to specify the current page - for example `?current_page=1` - we can chain the method `pageName()`: ```php -$source = new Request('GET', 'https://paginated-json-api.test?page_size=1'); - -$items = lazyJsonPages($source, $path, function (Config $config) { - $config->pages('total_pages')->perPage(500, 'page_size'); -}); +LazyJsonPages::from($source)->pageName('current_page'); ``` -Some APIs do not allow to request only 1 item per page, in these cases we can specify the number of items present on the first page as third argument: +Otherwise, if the number of the current page is present in the URI path - for example `https://example.com/users/1` - we can chain the method `pageInPath()`: ```php -$source = new Request('GET', 'https://paginated-json-api.test?page_size=5'); - -$items = lazyJsonPages($source, $path, function (Config $config) { - $config->pages('total_pages')->perPage(500, 'page_size', 5); -}); +LazyJsonPages::from($source)->pageInPath(); ``` -As always, we can either set the configuration through the `Config` object or with an associative array: +Some API paginations may start with a page different from `1`. If that's the case, we can define the first page by chaining the method `firstPage()`: ```php -$config = [ - 'pages' => 'total_pages', - 'per_page' => [500, 'page_size', 5], -]; +LazyJsonPages::from($source)->firstPage(0); ``` -From now on we will just use the object-oriented version for brevity. Also note that the "per page" strategy can be used with any of the configurations seen so far: +Now that we have customized the basic structure of the API, we can describe how items are paginated depending on whether the pagination is [length-aware](#-length-aware-paginations) or [cursor](#-cursor-and-next-page-paginations) based. -```php -$config->pages('total_pages')->perPage(500, 'page_size'); -// or -$config->items('total_items')->perPage(500, 'page_size'); -// or -$config->lastPage('last_page_key')->perPage(500, 'page_size'); -``` +### ๐Ÿ“ Length-aware paginations -### โ†ช๏ธ Cursor and next-page paginations +The term "length-aware" indicates any pagination containing at least one of the following length information: +- the total number of pages +- the total number of items +- the number of the last page -Some APIs show only the number or cursor of the next page in all pages. We can tackle this kind of pagination by indicating the JSON key holding the next page: +Lazy JSON Pages only needs one of these details to work properly: ```php -$config->nextPage('next_page_key'); -``` +LazyJsonPages::from($source)->totalPages('pagination.total_pages'); -The JSON key may hold a number, a cursor or a URL, Lazy JSON Pages supports all of them. +LazyJsonPages::from($source)->totalItems('pagination.total_items'); +LazyJsonPages::from($source)->lastPage('pagination.last_page'); +``` -### ๐Ÿ›  Requests fine-tuning +If the length information is nested in the JSON body, we can use dot-notation to indicate the level of nesting. For example, `pagination.total_pages` means that the total number of pages sits in the object `pagination`, under the key `total_pages`. -Lazy JSON Pages provides a number of settings to adjust the way HTTP requests are sent to fetch pages. For example pages can be requested in chunks, so that only a few streams are kept in memory at once: +Otherwise, if the length information is displayed in the headers, we can use the same methods to gather it by simply defining the name of the header: ```php -$config->chunk(3); +LazyJsonPages::from($source)->lastPage('X-Last-Page'); ``` -The configuration above fetches 3 pages concurrently, loads the paginated items into a lazy collection and proceeds with the next 3 pages. Chunking benefits memory usage at the expense of speed, no chunking is set by default but it is recommended when dealing with a lot of data. +APIs can expose their length information in the form of numbers (`total_pages: 10`) or URIs (`last_page: "https://example.com?page=10"`), Lazy JSON Pages supports both. -To minimize the memory usage Lazy JSON Pages can fetch pages synchronously, i.e. one by one, beware that this is also the slowest solution: +To save more memory when setting the total number of items, we can also define the number of items shown in each page: ```php -$config->sync(); +LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(20); ``` -We can also set how many HTTP requests we want to send concurrently. By default 10 pages are fetched asynchronously: +When dealing with a lot of data, it may be a good idea to fetch only 1 item on the first page and leverage the length information on that page to calculate the total number of pages/items without having to load all the other items of that page. + +We can do that by calling `perPage()` with: +- the number of items that we want to show per page (we can override the pagination default) +- the query parameter or header that holds the number of items per page ```php -$config->concurrency(25); +// indicate that the number of items per page is defined by the `limit` query parameter, e.g. ?limit=50 +LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(30, 'limit'); + +// indicate that the number of items per page is defined by the `X-Limit` header +LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(30, header: 'X-Limit'); ``` -Every HTTP request has a timeout of 5 seconds by default, but some APIs may be slow to respond. In this case we may need to set a higher timeout: +Some APIs may not allow to request only 1 item per page, in these cases we can specify how many items should be loaded on the first page as third argument: ```php -$config->timeout(15); +LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(30, 'limit', 5); + +LazyJsonPages::from($source) + ->totalItems('pagination.total_items') + ->perPage(30, header: 'X-Limit', firstPageItems: 5); ``` -When a request fails, it has up to 3 attempts to succeed. This number can of course be adjusted as needed: +We can leverage the `perPage()` strategy with all the length-aware methods seen before: ```php -$config->attempts(5); -``` +LazyJsonPages::from($source)->totalPages('pagination.total_pages')->perPage(30, 'limit'); -The backoff strategy allows us to wait some time before sending other requests when one page fails to be loaded. The package provides an exponential backoff by default, when a request fails it gets retried after 0, 1, 4, 9 seconds and so on. This strategy can also be overridden: +LazyJsonPages::from($source)->totalItems('pagination.total_items')->perPage(30, 'limit'); -```php -$config->backoff(function (int $attempt) { - return $attempt ** 2 * 100; -}); +LazyJsonPages::from($source)->lastPage('pagination.last_page')->perPage(30, 'limit'); ``` -The above backoff strategy will wait for 100, 400, 900 milliseconds and so on. -Putting all together, this is one of the possible configurations: +### โ†ช๏ธ Cursor and next-page paginations -```php -$source = new Request('GET', 'https://paginated-json-api.test?page_size=1'); - -$items = lazyJsonPages($source, 'data.results', function (Config $config) { - $config - ->pages('total_pages') - ->perPage(500, 'page_size') - ->chunk(3) - ->timeout(15) - ->attempts(5) - ->backoff(fn (int $attempt) => $attempt ** 2 * 100); -}); - -$items - ->filter(fn (array $item) => $this->isValid($item)) - ->map(fn (array $item) => $this->transform($item)) - ->each(fn (array $item) => $this->save($item)); -``` +> [!WARNING] +> The documentation of this feature is a work in progress. -### ๐Ÿ’ข Errors handling +### ๐Ÿš€ Requests optimization -As seen above, we can mitigate potentially faulty HTTP requests with backoffs, timeouts and retries. When we reach the maximum number of attempts and a request keeps failing, an `OutOfAttemptsException` is thrown. +> [!WARNING] +> The documentation of this feature is a work in progress. -When caught, this exception provides information about what went wrong, including the actual exception that was thrown, the pages that failed to be fetched and the paginated items that were loaded before the failure happened: -```php -use Cerbero\LazyJsonPages\Exceptions\OutOfAttemptsException; - -try { - $items = lazyJsonPages($source, $path, $config); -} catch (OutOfAttemptsException $e) { - // the actual exception that was thrown - $e->original; - // the pages that failed to be fetched - $e->failedPages; - // a LazyCollection with items loaded before the error - $e->items; -} -``` +### ๐Ÿ’ข Errors handling + +> [!WARNING] +> The documentation of this feature is a work in progress. ## ๐Ÿ“† Change log From 62f36516f50995d66f02c43a0cb66a420761bba7 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:43:21 +1000 Subject: [PATCH 037/109] Add concerns to resolve pages and yield items --- src/Concerns/ResolvesPages.php | 36 ++++++++++++++++++++ src/Concerns/YieldsPaginatedItems.php | 48 +++++++++++++++++++++++++++ src/Paginations/Pagination.php | 19 +++++++++-- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 src/Concerns/ResolvesPages.php create mode 100644 src/Concerns/YieldsPaginatedItems.php diff --git a/src/Concerns/ResolvesPages.php b/src/Concerns/ResolvesPages.php new file mode 100644 index 0000000..f436298 --- /dev/null +++ b/src/Concerns/ResolvesPages.php @@ -0,0 +1,36 @@ + (int) $value, + !is_string($value) || $value === '' => null, + !($query = parse_url($value, PHP_URL_QUERY)) => $onlyNumerics ? null : $value, + default => $this->pageFromQuery($query, $onlyNumerics), + }; + } + + /** + * Retrieve the page from the given query. + * + * @return ($onlyNumerics is true ? int|null : string|int|null) + */ + protected function pageFromQuery(string $query, bool $onlyNumerics = true): string|int|null + { + parse_str($query, $parameters); + + return $this->toPage($parameters[$this->config->pageName] ?? null, $onlyNumerics); + } +} diff --git a/src/Concerns/YieldsPaginatedItems.php b/src/Concerns/YieldsPaginatedItems.php new file mode 100644 index 0000000..40f4364 --- /dev/null +++ b/src/Concerns/YieldsPaginatedItems.php @@ -0,0 +1,48 @@ + + */ + protected function yieldItemsAndReturnKey(ResponseInterface $response, string $key): Generator + { + $pointers = [$this->config->pointer]; + + if (($value = $response->getHeaderLine($key)) === '') { + $pointers[DotsConverter::toPointer($key)] = fn(mixed $value) => (object) compact('value'); + } + + foreach (JsonParser::parse($response)->pointers($pointers) as $item) { + if (is_object($item)) { + $value = $item->value; + } else { + yield $item; + } + } + + return $value; + } + + /** + * Yield paginated items from the given source. + * + * @return Generator + */ + protected function yieldItemsFrom(mixed $source): Generator + { + yield from JsonParser::parse($source)->pointer($this->config->pointer); + } +} diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index b785968..c53c778 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -4,7 +4,10 @@ namespace Cerbero\LazyJsonPages\Paginations; +use Cerbero\LazyJsonPages\Concerns\ResolvesPages; +use Cerbero\LazyJsonPages\Concerns\YieldsPaginatedItems; use Cerbero\LazyJsonPages\Dtos\Config; +use Cerbero\LazyJsonPages\Services\Book; use Cerbero\LazyJsonPages\Sources\Source; use IteratorAggregate; use Traversable; @@ -12,10 +15,18 @@ /** * The abstract implementation of a pagination. * - * @implements IteratorAggregate + * @implements IteratorAggregate */ abstract class Pagination implements IteratorAggregate { + use YieldsPaginatedItems; + use ResolvesPages; + + /** + * The collector of pages. + */ + public readonly Book $book; + /** * Determine whether the configuration matches this pagination. */ @@ -24,12 +35,14 @@ abstract public function matches(): bool; /** * Yield the paginated items. * - * @return Traversable + * @return Traversable */ abstract public function getIterator(): Traversable; final public function __construct( protected readonly Source $source, protected readonly Config $config, - ) {} + ) { + $this->book = new Book(); + } } From 7b07a07234110c8c148bd8c38a29b7b5dea10973 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:43:48 +1000 Subject: [PATCH 038/109] Create exception --- src/Exceptions/InvalidPageException.php | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/Exceptions/InvalidPageException.php diff --git a/src/Exceptions/InvalidPageException.php b/src/Exceptions/InvalidPageException.php new file mode 100644 index 0000000..ee32664 --- /dev/null +++ b/src/Exceptions/InvalidPageException.php @@ -0,0 +1,21 @@ + Date: Sat, 20 Jan 2024 20:44:03 +1000 Subject: [PATCH 039/109] Update exception --- src/Exceptions/OutOfAttemptsException.php | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/Exceptions/OutOfAttemptsException.php b/src/Exceptions/OutOfAttemptsException.php index 3535e7e..ecfcf93 100644 --- a/src/Exceptions/OutOfAttemptsException.php +++ b/src/Exceptions/OutOfAttemptsException.php @@ -2,7 +2,7 @@ namespace Cerbero\LazyJsonPages\Exceptions; -use Cerbero\LazyJsonPages\Services\Outcome; +use Closure; use GuzzleHttp\Exception\TransferException; use Illuminate\Support\LazyCollection; @@ -11,13 +11,6 @@ */ class OutOfAttemptsException extends LazyJsonPagesException { - /** - * The pages that caused the failure. - * - * @var int[] - */ - public readonly array $failedPages; - /** * The paginated items loaded before the failure. * @@ -27,11 +20,13 @@ class OutOfAttemptsException extends LazyJsonPagesException /** * Instantiate the class. + * + * @param array $failedPages + * @param (Closure(): Generator) $items */ - public function __construct(TransferException $e, Outcome $outcome) + public function __construct(TransferException $e, public readonly array $failedPages, Closure $items) { - $this->failedPages = $outcome->pullFailedPages(); - $this->items = new LazyCollection(fn() => yield from $outcome->pullItems()); + $this->items = new LazyCollection($items); parent::__construct($e->getMessage(), 0, $e); } From 4a9758f85b93b45ba9e49d75c0844b62daa7d927 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:44:19 +1000 Subject: [PATCH 040/109] Update docblocks --- src/Paginations/AnyPagination.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index cd24518..13d92f3 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -8,7 +8,7 @@ use Traversable; /** - * The aggregator of paginations. + * The aggregator of supported paginations. */ class AnyPagination extends Pagination { @@ -39,7 +39,7 @@ public function matches(): bool /** * Yield the paginated items. * - * @return Traversable + * @return Traversable */ public function getIterator(): Traversable { From d53537dfae9beb0a1258db7b1a86afc537fef317 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:44:33 +1000 Subject: [PATCH 041/109] Create singleton client --- src/Services/Client.php | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/Services/Client.php diff --git a/src/Services/Client.php b/src/Services/Client.php new file mode 100644 index 0000000..2f3f7ed --- /dev/null +++ b/src/Services/Client.php @@ -0,0 +1,72 @@ + + */ + private static array $defaultOptions = [ + RequestOptions::STREAM => true, + RequestOptions::HEADERS => [ + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + ], + ]; + + /** + * The custom options. + */ + private static array $options = []; + + /** + * The Guzzle client instance. + */ + private static ?Guzzle $guzzle = null; + + /** + * Instantiate the class. + */ + private function __construct() + { + // disable the constructor + } + + /** + * Set the Guzzle client options. + */ + public static function configure(array $options): void + { + self::$options = array_replace_recursive(self::$options, $options); + } + + /** + * Retrieve the Guzzle client instance. + */ + public static function instance(): Guzzle + { + return self::$guzzle ??= new Guzzle( + array_replace_recursive(self::$defaultOptions, self::$options), + ); + } + + /** + * Clean up the static values. + */ + public static function reset(): void + { + self::$guzzle = null; + self::$options = []; + } +} From 38c4ad0c37781dba9dc566c382803b9dd16f9f04 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 20:52:56 +1000 Subject: [PATCH 042/109] Update readme --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 36df8bd..7878c66 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ $lazyCollection = LazyJsonPages::from($source) Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. -Need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. +> [!TIP] +> Need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. ## ๐Ÿ“ฆ Install @@ -36,9 +37,9 @@ composer require cerbero/lazy-json-pages * [๐Ÿ‘ฃ Basics](#-basics) * [๐Ÿ’ง Sources](#-sources) -* [๐Ÿ›๏ธ Pagination structure](#-pagination-structure) +* [๐Ÿ›๏ธ Pagination structure](#%EF%B8%8F-pagination-structure) * [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) -* [โ†ช๏ธ Cursor and next-page paginations](#-cursor-and-next-page-paginations) +* [โ†ช๏ธ Cursor and next-page paginations](#%EF%B8%8F-cursor-and-next-page-paginations) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) @@ -118,7 +119,7 @@ Some API paginations may start with a page different from `1`. If that's the cas LazyJsonPages::from($source)->firstPage(0); ``` -Now that we have customized the basic structure of the API, we can describe how items are paginated depending on whether the pagination is [length-aware](#-length-aware-paginations) or [cursor](#-cursor-and-next-page-paginations) based. +Now that we have customized the basic structure of the API, we can describe how items are paginated depending on whether the pagination is [length-aware](#-length-aware-paginations) or [cursor](#%EF%B8%8F-cursor-and-next-page-paginations) based. ### ๐Ÿ“ Length-aware paginations @@ -159,7 +160,7 @@ LazyJsonPages::from($source) When dealing with a lot of data, it may be a good idea to fetch only 1 item on the first page and leverage the length information on that page to calculate the total number of pages/items without having to load all the other items of that page. We can do that by calling `perPage()` with: -- the number of items that we want to show per page (we can override the pagination default) +- the number of items that we want to show per page (we can also override the pagination default) - the query parameter or header that holds the number of items per page ```php From f0d85935cb1d87eff909b9b0a0a072b80799c757 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 21:35:16 +1000 Subject: [PATCH 043/109] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7878c66..b546c37 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ When calling `collect()`, we indicate that the pagination structure is defined a ### ๐Ÿ’ง Sources -A source is any mean that can point to a paginated JSON API. A number of sources are supported by default: +A source is any mean that can point to a paginated JSON API. A number of sources is supported by default: - **endpoint URIs**, e.g. `https://example.com/api/v1/users` or any instance of `Psr\Http\Message\UriInterface` - **PSR-7 requests**, i.e. any instance of `Psr\Http\Message\RequestInterface` From ee100c9c00096994bbc6e03b51a6e8376fbe5991 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 20 Jan 2024 21:35:39 +1000 Subject: [PATCH 044/109] Remove the response value object --- src/LazyJsonPages.php | 3 +- src/ValueObjects/Response.php | 97 ----------------------------------- 2 files changed, 1 insertion(+), 99 deletions(-) delete mode 100644 src/ValueObjects/Response.php diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index f47d793..8ee8116 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -9,7 +9,6 @@ use Cerbero\LazyJsonPages\Paginations\AnyPagination; use Cerbero\LazyJsonPages\Services\Client; use Cerbero\LazyJsonPages\Sources\AnySource; -use Cerbero\LazyJsonPages\ValueObjects\Response; use Closure; use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; @@ -129,7 +128,7 @@ public function perPage(int $items, ?string $key = null, int $firstPageItems = 1 */ public function nextPage(Closure|string $key): self { - $this->config['nextPage'] = $key instanceof Closure ? $key : fn(Response $response) => $response->get($key); + $this->config['nextPage'] = $this->valueFromResponse($key); return $this; } diff --git a/src/ValueObjects/Response.php b/src/ValueObjects/Response.php deleted file mode 100644 index 9523604..0000000 --- a/src/ValueObjects/Response.php +++ /dev/null @@ -1,97 +0,0 @@ - $headers - */ - public readonly array $headers; - - /** - * Instantiate the class. - * - * @param array $headers - */ - public function __construct(public readonly string $json, array $headers) - { - $this->headers = $this->normalizeHeaders($headers); - } - - /** - * Normalize the given headers. - * - * @param array $headers - * @return array - */ - private function normalizeHeaders(array $headers): array - { - $normalizedHeaders = []; - - foreach ($headers as $name => $value) { - $normalizedHeaders[strtolower($name)] = $value; - } - - return $normalizedHeaders; - } - - /** - * Retrieve a value from the body or a header. - */ - public function get(string $key): mixed - { - return $this->hasHeader($key) ? $this->header($key) : $this->json($key); - } - - /** - * Determine whether the given header is set. - */ - public function hasHeader(string $header): bool - { - return isset($this->headers[strtolower($header)]); - } - - /** - * Retrieve the given header. - * - * @return ?string[] - */ - public function header(string $header): ?array - { - return $this->headers[strtolower($header)] ?? null; - } - - /** - * Retrieve a value from the body. - */ - public function json(string $key): mixed - { - $pointer = DotsConverter::toPointer($key); - $array = JsonParser::parse($this->json)->pointer($pointer)->toArray(); - - return empty($array) ? null : current($array); - } - - /** - * Retrieve an iterator with the given JSON pointer. - * - * @return Traversable - */ - public function pointer(string $pointer): Traversable - { - /** @var Traversable */ - return JsonParser::parse($this->json)->pointer($pointer); - } -} From 716e0ead1c0c734fa12f769d96a6668e8338b2a0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 21 Jan 2024 23:47:03 +1000 Subject: [PATCH 045/109] Create fixtures --- tests/fixtures/items.php | 18 ++++++++++++++++++ tests/fixtures/totalPages/page1.json | 22 ++++++++++++++++++++++ tests/fixtures/totalPages/page2.json | 22 ++++++++++++++++++++++ tests/fixtures/totalPages/page3.json | 19 +++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 tests/fixtures/items.php create mode 100644 tests/fixtures/totalPages/page1.json create mode 100644 tests/fixtures/totalPages/page2.json create mode 100644 tests/fixtures/totalPages/page3.json diff --git a/tests/fixtures/items.php b/tests/fixtures/items.php new file mode 100644 index 0000000..fb15a2c --- /dev/null +++ b/tests/fixtures/items.php @@ -0,0 +1,18 @@ + 'item1'], + ['name' => 'item2'], + ['name' => 'item3'], + ['name' => 'item4'], + ['name' => 'item5'], + ['name' => 'item6'], + ['name' => 'item7'], + ['name' => 'item8'], + ['name' => 'item9'], + ['name' => 'item10'], + ['name' => 'item11'], + ['name' => 'item12'], + ['name' => 'item13'], + ['name' => 'item14'], +]; diff --git a/tests/fixtures/totalPages/page1.json b/tests/fixtures/totalPages/page1.json new file mode 100644 index 0000000..e24a6b0 --- /dev/null +++ b/tests/fixtures/totalPages/page1.json @@ -0,0 +1,22 @@ +{ + "data": [ + { + "name": "item1" + }, + { + "name": "item2" + }, + { + "name": "item3" + }, + { + "name": "item4" + }, + { + "name": "item5" + } + ], + "meta": { + "total_pages": 3 + } +} diff --git a/tests/fixtures/totalPages/page2.json b/tests/fixtures/totalPages/page2.json new file mode 100644 index 0000000..4ebb787 --- /dev/null +++ b/tests/fixtures/totalPages/page2.json @@ -0,0 +1,22 @@ +{ + "data": [ + { + "name": "item6" + }, + { + "name": "item7" + }, + { + "name": "item8" + }, + { + "name": "item9" + }, + { + "name": "item10" + } + ], + "meta": { + "total_pages": 3 + } +} diff --git a/tests/fixtures/totalPages/page3.json b/tests/fixtures/totalPages/page3.json new file mode 100644 index 0000000..f308f95 --- /dev/null +++ b/tests/fixtures/totalPages/page3.json @@ -0,0 +1,19 @@ +{ + "data": [ + { + "name": "item11" + }, + { + "name": "item12" + }, + { + "name": "item13" + }, + { + "name": "item14" + } + ], + "meta": { + "total_pages": 3 + } +} From 5fb3385a176bdf07a7820e0f56dd7a07b4ee2d8f Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 21 Jan 2024 23:47:24 +1000 Subject: [PATCH 046/109] Add dataset --- tests/Feature/Datasets.php | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/Feature/Datasets.php diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php new file mode 100644 index 0000000..53c6c1c --- /dev/null +++ b/tests/Feature/Datasets.php @@ -0,0 +1,11 @@ + Date: Sun, 21 Jan 2024 23:48:03 +1000 Subject: [PATCH 047/109] Add helpers and custom expectations --- tests/Pest.php | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/Pest.php b/tests/Pest.php index baddc84..09d2ddd 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,5 +1,11 @@ extend('toBeOne', function () { -// return $this->toBe(1); -// }); +expect()->extend('toLoadItemsViaRequests', function (array $items, array $requests) { + $responses = $transactions = $expectedUris = []; + + foreach ($requests as $uri => $fixture) { + $responses[] = new Response(body: file_get_contents(fixture($fixture))); + $expectedUris[] = $uri; + } + + $stack = HandlerStack::create(new MockHandler($responses)); + + $stack->push(Middleware::history($transactions)); + + Client::configure(['handler' => $stack]); + + $this->sequence(...$items); + + $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); + + expect($actualUris)->toBe($expectedUris); +}); /* |-------------------------------------------------------------------------- @@ -39,7 +62,6 @@ | */ -// function something() -// { -// // .. -// } +function fixture(string $filename) { + return __DIR__ . "/fixtures/{$filename}"; +} From 9d9392fcb8777204a5f09e7f59c64642ee5bbd81 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 21 Jan 2024 23:48:17 +1000 Subject: [PATCH 048/109] Create test for sources --- tests/Feature/SourceTest.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/Feature/SourceTest.php diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php new file mode 100644 index 0000000..fda6fea --- /dev/null +++ b/tests/Feature/SourceTest.php @@ -0,0 +1,16 @@ +totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/api/v1/users' => 'totalPages/page1.json', + 'https://example.com/api/v1/users?page=2' => 'totalPages/page2.json', + 'https://example.com/api/v1/users?page=3' => 'totalPages/page3.json', + ]); +})->with('sources'); From e5ced02d35f1e92ff93c4b75b669bddd9483d710 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 22 Jan 2024 19:34:53 +1000 Subject: [PATCH 049/109] Create exception --- src/Exceptions/InvalidPageException.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Exceptions/InvalidPageException.php b/src/Exceptions/InvalidPageException.php index ee32664..51191f8 100644 --- a/src/Exceptions/InvalidPageException.php +++ b/src/Exceptions/InvalidPageException.php @@ -2,10 +2,6 @@ namespace Cerbero\LazyJsonPages\Exceptions; -use Cerbero\LazyJsonPages\Services\Outcome; -use GuzzleHttp\Exception\TransferException; -use Illuminate\Support\LazyCollection; - /** * The exception thrown when a given JSON key does not contain a valid page. */ From c3fb21641bcdef7121ba35949b03518fae1c7d21 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 22 Jan 2024 19:36:55 +1000 Subject: [PATCH 050/109] Implement supported sources --- README.md | 2 + src/Exceptions/RequestNotSentException.php | 17 ++++++++ src/Services/Client.php | 24 +++++------ src/Sources/AnySource.php | 19 +++----- src/Sources/CustomSource.php | 42 ++++++++++++++++++ src/Sources/Endpoint.php | 2 +- src/Sources/LaravelClientRequest.php | 44 +++++++++++++++++++ src/Sources/LaravelClientResponse.php | 46 ++++++++++++++++++++ src/Sources/Psr7Request.php | 43 +++++++++++++++++++ src/Sources/Source.php | 21 +++++---- src/Sources/SymfonyRequest.php | 50 ++++++++++++++++++++++ tests/Feature/Datasets.php | 27 +++++++++--- tests/Feature/SourceTest.php | 6 +-- tests/Sources/CustomSourceSample.php | 34 +++++++++++++++ 14 files changed, 332 insertions(+), 45 deletions(-) create mode 100644 src/Exceptions/RequestNotSentException.php create mode 100644 src/Sources/CustomSource.php create mode 100644 src/Sources/LaravelClientRequest.php create mode 100644 src/Sources/LaravelClientResponse.php create mode 100644 src/Sources/Psr7Request.php create mode 100644 src/Sources/SymfonyRequest.php create mode 100644 tests/Sources/CustomSourceSample.php diff --git a/README.md b/README.md index b546c37..86a0c76 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ A source is any mean that can point to a paginated JSON API. A number of sources - **PSR-7 requests**, i.e. any instance of `Psr\Http\Message\RequestInterface` - **Laravel HTTP client requests**, i.e. any instance of `Illuminate\Http\Client\Request` - **Laravel HTTP client responses**, i.e. any instance of `Illuminate\Http\Client\Response` +- **Laravel HTTP requests**, i.e. any instance of `Illuminate\Http\Request` +- **Symfony requests**, i.e. any instance of `Symfony\Component\HttpFoundation\Request` - **user-defined sources**, i.e. any instance of `Cerbero\LazyJsonPages\Sources\Source` Here are some examples of sources: diff --git a/src/Exceptions/RequestNotSentException.php b/src/Exceptions/RequestNotSentException.php new file mode 100644 index 0000000..23ca0bb --- /dev/null +++ b/src/Exceptions/RequestNotSentException.php @@ -0,0 +1,17 @@ +[] */ protected array $supportedSources = [ - // CustomSource::class, + CustomSource::class, Endpoint::class, - // LaravelClientRequest::class, - // LaravelClientResponse::class, - // LaravelRequest::class, - // Psr7Request::class, - // SymfonyRequest::class, + LaravelClientRequest::class, + LaravelClientResponse::class, + Psr7Request::class, + SymfonyRequest::class, ]; /** @@ -31,14 +30,6 @@ class AnySource extends Source */ protected ?Source $matchingSource; - /** - * Determine whether this class can handle the source. - */ - public function matches(): bool - { - return true; - } - /** * Retrieve the HTTP request. */ diff --git a/src/Sources/CustomSource.php b/src/Sources/CustomSource.php new file mode 100644 index 0000000..3970655 --- /dev/null +++ b/src/Sources/CustomSource.php @@ -0,0 +1,42 @@ +source instanceof Source; + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return $this->source->request(); + } + + /** + * Retrieve the HTTP response. + * + * @return ResponseInterface + */ + public function response(): ResponseInterface + { + return $this->source->response(); + } +} diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index e328a57..7eb81a9 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -36,7 +36,7 @@ class Endpoint extends Source public function matches(): bool { return $this->source instanceof UriInterface - || $this->isEndpoint($this->source); + || (is_string($this->source) && $this->isEndpoint($this->source)); } /** diff --git a/src/Sources/LaravelClientRequest.php b/src/Sources/LaravelClientRequest.php new file mode 100644 index 0000000..072674c --- /dev/null +++ b/src/Sources/LaravelClientRequest.php @@ -0,0 +1,44 @@ +source instanceof Request; + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return $this->source->toPsrRequest(); + } + + /** + * Retrieve the HTTP response. + * + * @return ResponseInterface + */ + public function response(): ResponseInterface + { + return Client::instance()->send($this->request()); + } +} diff --git a/src/Sources/LaravelClientResponse.php b/src/Sources/LaravelClientResponse.php new file mode 100644 index 0000000..bed169b --- /dev/null +++ b/src/Sources/LaravelClientResponse.php @@ -0,0 +1,46 @@ +source instanceof Response; + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return $this->source->transferStats?->getRequest() + ?: throw new RequestNotSentException(); + } + + /** + * Retrieve the HTTP response. + * + * @return ResponseInterface + */ + public function response(): ResponseInterface + { + return $this->source->toPsrResponse(); + } +} diff --git a/src/Sources/Psr7Request.php b/src/Sources/Psr7Request.php new file mode 100644 index 0000000..35821d9 --- /dev/null +++ b/src/Sources/Psr7Request.php @@ -0,0 +1,43 @@ +source instanceof RequestInterface; + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return $this->source; + } + + /** + * Retrieve the HTTP response. + * + * @return ResponseInterface + */ + public function response(): ResponseInterface + { + return Client::instance()->send($this->source); + } +} diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 6ad9fae..c69d95d 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -12,15 +12,6 @@ */ abstract class Source { - final public function __construct( - protected readonly mixed $source, - ) {} - - /** - * Determine whether this class can handle the source. - */ - abstract public function matches(): bool; - /** * Retrieve the HTTP request. */ @@ -32,4 +23,16 @@ abstract public function request(): RequestInterface; * @return ResponseInterface */ abstract public function response(): ResponseInterface; + + final public function __construct( + protected readonly mixed $source, + ) {} + + /** + * Determine whether this class can handle the source. + */ + public function matches(): bool + { + return true; + } } diff --git a/src/Sources/SymfonyRequest.php b/src/Sources/SymfonyRequest.php new file mode 100644 index 0000000..69166c5 --- /dev/null +++ b/src/Sources/SymfonyRequest.php @@ -0,0 +1,50 @@ +source instanceof Request; + } + + /** + * Retrieve the HTTP request. + */ + public function request(): RequestInterface + { + return new Psr7Request( + $this->source->getMethod(), + $this->source->getUri(), + $this->source->headers->all(), + $this->source->getContent() ?: null, + ); + } + + /** + * Retrieve the HTTP response. + * + * @return ResponseInterface + */ + public function response(): ResponseInterface + { + return Client::instance()->send($this->request()); + } +} diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index 53c6c1c..e0f6756 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -1,11 +1,26 @@ transferStats = new TransferStats($psr7Request, $psr7Response); - foreach ($sources as $source) { - yield $source; - } + yield 'user-defined source' => [new CustomSourceSample(null), false]; + yield 'endpoint' => [$uri, true]; + yield 'Laravel client request' => [new LaravelClientRequest($psr7Request), true]; + yield 'Laravel client response' => [$laravelClientResponse, false]; + yield 'Laravel request' => [LaravelRequest::create($uri), true]; + yield 'PSR-7 request' => [$psr7Request, true]; + yield 'Symfony request' => [SymfonyRequest::create($uri), true]; }); diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php index fda6fea..27a6134 100644 --- a/tests/Feature/SourceTest.php +++ b/tests/Feature/SourceTest.php @@ -2,14 +2,14 @@ use Cerbero\LazyJsonPages\LazyJsonPages; -it('supports multiple sources', function (mixed $source) { - $expectedItems = require_once fixture('items.php'); +it('supports multiple sources', function (mixed $source, bool $requestsFirstPage) { + $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from($source) ->totalPages('meta.total_pages') ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ - 'https://example.com/api/v1/users' => 'totalPages/page1.json', + ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'totalPages/page1.json'] : [], 'https://example.com/api/v1/users?page=2' => 'totalPages/page2.json', 'https://example.com/api/v1/users?page=3' => 'totalPages/page3.json', ]); diff --git a/tests/Sources/CustomSourceSample.php b/tests/Sources/CustomSourceSample.php new file mode 100644 index 0000000..90c317a --- /dev/null +++ b/tests/Sources/CustomSourceSample.php @@ -0,0 +1,34 @@ + Date: Tue, 23 Jan 2024 23:51:26 +1000 Subject: [PATCH 051/109] Move fixtures --- tests/Feature/Datasets.php | 2 +- tests/Feature/SourceTest.php | 6 +++--- tests/Sources/CustomSourceSample.php | 2 +- tests/fixtures/{totalPages => lengthAware}/page1.json | 3 ++- tests/fixtures/{totalPages => lengthAware}/page2.json | 3 ++- tests/fixtures/{totalPages => lengthAware}/page3.json | 3 ++- 6 files changed, 11 insertions(+), 8 deletions(-) rename tests/fixtures/{totalPages => lengthAware}/page1.json (84%) rename tests/fixtures/{totalPages => lengthAware}/page2.json (84%) rename tests/fixtures/{totalPages => lengthAware}/page3.json (82%) diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index e0f6756..e9b15e7 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -12,7 +12,7 @@ dataset('sources', function () { $uri = 'https://example.com/api/v1/users'; $psr7Request = new Psr7Request('GET', $uri); - $psr7Response = new Psr7Response(body: file_get_contents(fixture('totalPages/page1.json'))); + $psr7Response = new Psr7Response(body: file_get_contents(fixture('lengthAware/page1.json'))); $laravelClientResponse = new LaravelClientResponse($psr7Response); $laravelClientResponse->transferStats = new TransferStats($psr7Request, $psr7Response); diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php index 27a6134..d25d349 100644 --- a/tests/Feature/SourceTest.php +++ b/tests/Feature/SourceTest.php @@ -9,8 +9,8 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ - ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'totalPages/page1.json'] : [], - 'https://example.com/api/v1/users?page=2' => 'totalPages/page2.json', - 'https://example.com/api/v1/users?page=3' => 'totalPages/page3.json', + ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'lengthAware/page1.json'] : [], + 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', ]); })->with('sources'); diff --git a/tests/Sources/CustomSourceSample.php b/tests/Sources/CustomSourceSample.php index 90c317a..1cc4d37 100644 --- a/tests/Sources/CustomSourceSample.php +++ b/tests/Sources/CustomSourceSample.php @@ -29,6 +29,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return new Response(body: file_get_contents(fixture('totalPages/page1.json'))); + return new Response(body: file_get_contents(fixture('lengthAware/page1.json'))); } } diff --git a/tests/fixtures/totalPages/page1.json b/tests/fixtures/lengthAware/page1.json similarity index 84% rename from tests/fixtures/totalPages/page1.json rename to tests/fixtures/lengthAware/page1.json index e24a6b0..55c3fb4 100644 --- a/tests/fixtures/totalPages/page1.json +++ b/tests/fixtures/lengthAware/page1.json @@ -17,6 +17,7 @@ } ], "meta": { - "total_pages": 3 + "total_pages": 3, + "total_items": 14 } } diff --git a/tests/fixtures/totalPages/page2.json b/tests/fixtures/lengthAware/page2.json similarity index 84% rename from tests/fixtures/totalPages/page2.json rename to tests/fixtures/lengthAware/page2.json index 4ebb787..0f13b6b 100644 --- a/tests/fixtures/totalPages/page2.json +++ b/tests/fixtures/lengthAware/page2.json @@ -17,6 +17,7 @@ } ], "meta": { - "total_pages": 3 + "total_pages": 3, + "total_items": 14 } } diff --git a/tests/fixtures/totalPages/page3.json b/tests/fixtures/lengthAware/page3.json similarity index 82% rename from tests/fixtures/totalPages/page3.json rename to tests/fixtures/lengthAware/page3.json index f308f95..9199d0a 100644 --- a/tests/fixtures/totalPages/page3.json +++ b/tests/fixtures/lengthAware/page3.json @@ -14,6 +14,7 @@ } ], "meta": { - "total_pages": 3 + "total_pages": 3, + "total_items": 14 } } From 1a70b47f0a1a3d7d2f8681883d83376e9a7e7130 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 23 Jan 2024 23:53:04 +1000 Subject: [PATCH 052/109] Rename exception and methods name --- ...eException.php => InvalidKeyException.php} | 6 ++--- src/Paginations/LengthAwarePagination.php | 22 +++++++++---------- src/Paginations/TotalPagesAwarePagination.php | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) rename src/Exceptions/{InvalidPageException.php => InvalidKeyException.php} (77%) diff --git a/src/Exceptions/InvalidPageException.php b/src/Exceptions/InvalidKeyException.php similarity index 77% rename from src/Exceptions/InvalidPageException.php rename to src/Exceptions/InvalidKeyException.php index 51191f8..536b778 100644 --- a/src/Exceptions/InvalidPageException.php +++ b/src/Exceptions/InvalidKeyException.php @@ -3,15 +3,15 @@ namespace Cerbero\LazyJsonPages\Exceptions; /** - * The exception thrown when a given JSON key does not contain a valid page. + * The exception thrown when a given JSON key does not contain a valid value. */ -class InvalidPageException extends LazyJsonPagesException +class InvalidKeyException extends LazyJsonPagesException { /** * Instantiate the class. */ public function __construct(public string $key) { - parent::__construct("The key [{$key}] does not contain a valid page."); + parent::__construct("The key [{$key}] does not contain a valid value."); } } diff --git a/src/Paginations/LengthAwarePagination.php b/src/Paginations/LengthAwarePagination.php index 025f2a1..adc5c0c 100644 --- a/src/Paginations/LengthAwarePagination.php +++ b/src/Paginations/LengthAwarePagination.php @@ -5,7 +5,7 @@ namespace Cerbero\LazyJsonPages\Paginations; use Cerbero\LazyJsonPages\Concerns\SendsAsyncRequests; -use Cerbero\LazyJsonPages\Exceptions\InvalidPageException; +use Cerbero\LazyJsonPages\Exceptions\InvalidKeyException; use Closure; use Generator; use Illuminate\Support\LazyCollection; @@ -24,35 +24,35 @@ abstract class LengthAwarePagination extends Pagination * @param (Closure(int): int)|null $callback * @return Generator */ - protected function yieldItemsUntilPage(string $key, ?Closure $callback = null): Generator + protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): Generator { yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $key); $page = $this->toPage($generator->getReturn()); if (!is_int($page)) { - throw new InvalidPageException($key); + throw new InvalidKeyException($key); } $page = $callback ? $callback($page) : $page; - foreach ($this->yieldPageResponsesUntil($page) as $pageResponse) { - yield from $this->yieldItemsFrom($pageResponse); - } + yield from $this->yieldItemsUntilPage($page); } /** - * Yield the HTTP page responses until given page. + * Yield paginated items until the given page is reached. * - * @return Generator + * @return Generator */ - protected function yieldPageResponsesUntil(int $page, ?UriInterface $uri = null): Generator + protected function yieldItemsUntilPage(int $page, ?UriInterface $uri = null): Generator { $uri ??= $this->source->request()->getUri(); $firstPageAlreadyFetched = strval($uri) == strval($this->source->request()->getUri()); $chunkedPages = $this->chunkPages($page, $firstPageAlreadyFetched); - yield from $this->fetchPagesAsynchronously($chunkedPages, $uri); + foreach ($this->fetchPagesAsynchronously($chunkedPages, $uri) as $page) { + yield from $this->yieldItemsFrom($page); + } } /** @@ -62,7 +62,7 @@ protected function yieldPageResponsesUntil(int $page, ?UriInterface $uri = null) */ protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection { - if ($pages == 1 && $shouldSkipFirstPage) { + if ($pages == 0 || ($pages == 1 && $shouldSkipFirstPage)) { return LazyCollection::empty(); } diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index 462f662..b8f9ba3 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -27,6 +27,6 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsUntilPage($this->config->totalPagesKey); + yield from $this->yieldItemsUntilKey($this->config->totalPagesKey); } } From d54561db5d6df9bacaaae1d58f82ecc9ed08f7da Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 23 Jan 2024 23:53:44 +1000 Subject: [PATCH 053/109] Update configuration --- src/Dtos/Config.php | 5 ++--- src/LazyJsonPages.php | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index f09e3a6..9a9e622 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -9,12 +9,11 @@ final class Config { public function __construct( - public readonly string $dot = "*", - public readonly string $pointer = '', + public readonly string $pointer, public readonly string $pageName = 'page', public readonly int $firstPage = 1, public readonly ?string $totalPagesKey = null, - public readonly ?int $totalItems = null, + public readonly ?string $totalItemsKey = null, public readonly ?int $perPage = null, public readonly ?string $perPageKey = null, public readonly ?int $perPageOverride = null, diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 8ee8116..1e4421b 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -104,9 +104,9 @@ private function valueFromResponse(Closure|string $key): mixed /** * Set the total number of items. */ - public function totalItems(Closure|string $totalItems): self + public function totalItems(string $key): self { - $this->config['totalItems'] = $this->integerFromResponse($totalItems); + $this->config['totalItemsKey'] = $key; return $this; } @@ -209,7 +209,6 @@ public function backoff(Closure $callback): self */ public function collect(string $dot = '*'): LazyCollection { - $this->config['dot'] = $dot; $this->config['pointer'] = DotsConverter::toPointer($dot); Client::configure($this->requestOptions); From e762f75e69266acf09cd84064d81687bac14fc48 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 23 Jan 2024 23:55:25 +1000 Subject: [PATCH 054/109] Support paginations aware of their total items --- README.md | 8 ---- src/Paginations/AnyPagination.php | 2 +- src/Paginations/TotalItemsAwarePagination.php | 48 +++++++++++++++++++ tests/Feature/PaginationTest.php | 29 +++++++++++ 4 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 src/Paginations/TotalItemsAwarePagination.php create mode 100644 tests/Feature/PaginationTest.php diff --git a/README.md b/README.md index 86a0c76..c6d4176 100644 --- a/README.md +++ b/README.md @@ -151,14 +151,6 @@ LazyJsonPages::from($source)->lastPage('X-Last-Page'); APIs can expose their length information in the form of numbers (`total_pages: 10`) or URIs (`last_page: "https://example.com?page=10"`), Lazy JSON Pages supports both. -To save more memory when setting the total number of items, we can also define the number of items shown in each page: - -```php -LazyJsonPages::from($source) - ->totalItems('pagination.total_items') - ->perPage(20); -``` - When dealing with a lot of data, it may be a good idea to fetch only 1 item on the first page and leverage the length information on that page to calculate the total number of pages/items without having to load all the other items of that page. We can do that by calling `perPage()` with: diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 13d92f3..2c01700 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -24,7 +24,7 @@ class AnyPagination extends Pagination // LimitPagination::class, // LinkHeaderPagination::class, // OffsetPagination::class, - // TotalItemsAwarePagination::class, + TotalItemsAwarePagination::class, TotalPagesAwarePagination::class, ]; diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php new file mode 100644 index 0000000..025e7fd --- /dev/null +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -0,0 +1,48 @@ +config->totalItemsKey !== null + && $this->config->totalPagesKey === null + && $this->config->perPage === null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + $perPage = 0; + $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->totalItemsKey); + + foreach ($generator as $item) { + yield $item; + ++$perPage; + } + + if (!is_numeric($totalItems = $generator->getReturn())) { + throw new InvalidKeyException($this->config->totalItemsKey); + } + + $totalPages = $perPage > 0 ? (int) ceil(intval($totalItems) / $perPage) : 0; + + yield from $this->yieldItemsUntilPage($totalPages); + } +} diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php new file mode 100644 index 0000000..db8ddc7 --- /dev/null +++ b/tests/Feature/PaginationTest.php @@ -0,0 +1,29 @@ +totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + ]); +}); + +it('supports paginations aware of their total items', function () { + $expectedItems = require fixture('items.php'); + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->totalItems('meta.total_items') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + ]); +}); From 45fafb44533126e05ab49066fa346e970bd28447 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 27 Jan 2024 17:18:53 +1000 Subject: [PATCH 055/109] Support paginations with current page in URI path --- README.md | 8 +++- src/Concerns/ResolvesPages.php | 36 ++++++++++++--- src/Concerns/SendsAsyncRequests.php | 4 +- src/Dtos/Config.php | 1 + src/Exceptions/InvalidPageInPathException.php | 17 +++++++ src/LazyJsonPages.php | 10 +++++ tests/Feature/StructureTest.php | 44 +++++++++++++++++++ 7 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 src/Exceptions/InvalidPageInPathException.php create mode 100644 tests/Feature/StructureTest.php diff --git a/README.md b/README.md index c6d4176..636fe64 100644 --- a/README.md +++ b/README.md @@ -109,12 +109,18 @@ If the API uses a query parameter different from `page` to specify the current p LazyJsonPages::from($source)->pageName('current_page'); ``` -Otherwise, if the number of the current page is present in the URI path - for example `https://example.com/users/1` - we can chain the method `pageInPath()`: +Otherwise, if the number of the current page is present in the URI path - for example `https://example.com/users/page/1` - we can chain the method `pageInPath()`: ```php LazyJsonPages::from($source)->pageInPath(); ``` +By default the last integer in the URI path is considered the page number. However we can customize the regular expression used to capture the page number, if need be: + +```php +LazyJsonPages::from($source)->pageInPath('~/page/(\d+)$~'); +``` + Some API paginations may start with a page different from `1`. If that's the case, we can define the first page by chaining the method `firstPage()`: ```php diff --git a/src/Concerns/ResolvesPages.php b/src/Concerns/ResolvesPages.php index f436298..4b92eb2 100644 --- a/src/Concerns/ResolvesPages.php +++ b/src/Concerns/ResolvesPages.php @@ -2,6 +2,10 @@ namespace Cerbero\LazyJsonPages\Concerns; +use Cerbero\LazyJsonPages\Exceptions\InvalidPageInPathException; +use GuzzleHttp\Psr7\Uri; +use Psr\Http\Message\UriInterface; + /** * The trait to resolve pages. */ @@ -17,20 +21,42 @@ protected function toPage(mixed $value, bool $onlyNumerics = true): string|int|n return match (true) { is_numeric($value) => (int) $value, !is_string($value) || $value === '' => null, - !($query = parse_url($value, PHP_URL_QUERY)) => $onlyNumerics ? null : $value, - default => $this->pageFromQuery($query, $onlyNumerics), + !$parsedUri = parse_url($value) => $onlyNumerics ? null : $value, + default => $this->pageFromParsedUri($parsedUri, $onlyNumerics), }; } /** - * Retrieve the page from the given query. + * Retrieve the page from the given parsed URI. * * @return ($onlyNumerics is true ? int|null : string|int|null) */ - protected function pageFromQuery(string $query, bool $onlyNumerics = true): string|int|null + protected function pageFromParsedUri(array $parsedUri, bool $onlyNumerics = true): string|int|null { - parse_str($query, $parameters); + if ($pattern = $this->config->pageInPath) { + preg_match($pattern, $parsedUri['path'] ?? '', $matches); + + return $this->toPage($matches[1] ?? null, $onlyNumerics); + } + + parse_str($parsedUri['query'] ?? '', $parameters); return $this->toPage($parameters[$this->config->pageName] ?? null, $onlyNumerics); } + + /** + * Retrieve the URI for the given page. + */ + protected function uriForPage(UriInterface $uri, string $page): UriInterface + { + if (!$pattern = $this->config->pageInPath) { + return Uri::withQueryValue($uri, $this->config->pageName, $page); + } + + if (!preg_match($pattern, $path = $uri->getPath(), $matches, PREG_OFFSET_CAPTURE)) { + throw new InvalidPageInPathException($path, $pattern); + } + + return $uri->withPath(substr_replace($path, $page, $matches[1][1], strlen($matches[1][0]))); + } } diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 7b4df3b..2a15dec 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -63,9 +63,7 @@ protected function yieldRequests(UriInterface $uri, array $pages): Generator $pages = $this->book->pullFailedPages() ?: $pages; foreach ($pages as $page) { - $pageUri = Uri::withQueryValue($uri, $this->config->pageName, (string) $page); - - yield $page => $request->withUri($pageUri); + yield $page => $request->withUri($this->uriForPage($uri, (string) $page)); } } } diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 9a9e622..9f085ca 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -11,6 +11,7 @@ final class Config public function __construct( public readonly string $pointer, public readonly string $pageName = 'page', + public readonly ?string $pageInPath = null, public readonly int $firstPage = 1, public readonly ?string $totalPagesKey = null, public readonly ?string $totalItemsKey = null, diff --git a/src/Exceptions/InvalidPageInPathException.php b/src/Exceptions/InvalidPageInPathException.php new file mode 100644 index 0000000..a70e106 --- /dev/null +++ b/src/Exceptions/InvalidPageInPathException.php @@ -0,0 +1,17 @@ +config['pageInPath'] = $pattern; + + return $this; + } + /** * Set the number of the first page. */ diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php new file mode 100644 index 0000000..8146cfb --- /dev/null +++ b/tests/Feature/StructureTest.php @@ -0,0 +1,44 @@ +pageInPath() + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/api/v1/users/page/1' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users/page/2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users/page/3' => 'lengthAware/page3.json', + ]); +}); + +it('supports a custom pattern for paginations with the current page in the URI path', function () { + $expectedItems = require fixture('items.php'); + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users/page1') + ->pageInPath('~/page(\d+)$~') + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/api/v1/users/page1' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users/page2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users/page3' => 'lengthAware/page3.json', + ]); +}); + +it('fails if it cannot capture the current page in the URI path', function () { + $expectedItems = require fixture('items.php'); + $lazyCollection = LazyJsonPages::from('https://example.com/users') + ->pageInPath() + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + 'https://example.com/users' => 'lengthAware/page1.json', + ]); +})->throws(InvalidPageInPathException::class, 'The pattern [/(\d+)(?!.*\d)/] could not capture any page from the path [/users].'); From 4c23bb9cc9930fccfe84ceb638c6c63d28e47381 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 27 Jan 2024 17:23:18 +1000 Subject: [PATCH 056/109] Expect items implicitly --- tests/Feature/PaginationTest.php | 6 ++---- tests/Feature/SourceTest.php | 3 +-- tests/Feature/StructureTest.php | 9 +++------ tests/Pest.php | 4 ++-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index db8ddc7..7013a45 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -3,12 +3,11 @@ use Cerbero\LazyJsonPages\LazyJsonPages; it('supports paginations aware of their total pages', function () { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->totalPages('meta.total_pages') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ 'https://example.com/api/v1/users' => 'lengthAware/page1.json', 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', @@ -16,12 +15,11 @@ }); it('supports paginations aware of their total items', function () { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->totalItems('meta.total_items') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ 'https://example.com/api/v1/users' => 'lengthAware/page1.json', 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php index d25d349..f68374f 100644 --- a/tests/Feature/SourceTest.php +++ b/tests/Feature/SourceTest.php @@ -3,12 +3,11 @@ use Cerbero\LazyJsonPages\LazyJsonPages; it('supports multiple sources', function (mixed $source, bool $requestsFirstPage) { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from($source) ->totalPages('meta.total_pages') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'lengthAware/page1.json'] : [], 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index 8146cfb..22d4d75 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -4,13 +4,12 @@ use Cerbero\LazyJsonPages\LazyJsonPages; it('supports paginations with the current page in the URI path', function () { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users/page/1') ->pageInPath() ->totalPages('meta.total_pages') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ 'https://example.com/api/v1/users/page/1' => 'lengthAware/page1.json', 'https://example.com/api/v1/users/page/2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users/page/3' => 'lengthAware/page3.json', @@ -18,13 +17,12 @@ }); it('supports a custom pattern for paginations with the current page in the URI path', function () { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users/page1') ->pageInPath('~/page(\d+)$~') ->totalPages('meta.total_pages') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ 'https://example.com/api/v1/users/page1' => 'lengthAware/page1.json', 'https://example.com/api/v1/users/page2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users/page3' => 'lengthAware/page3.json', @@ -32,13 +30,12 @@ }); it('fails if it cannot capture the current page in the URI path', function () { - $expectedItems = require fixture('items.php'); $lazyCollection = LazyJsonPages::from('https://example.com/users') ->pageInPath() ->totalPages('meta.total_pages') ->collect('data.*'); - expect($lazyCollection)->toLoadItemsViaRequests($expectedItems, [ + expect($lazyCollection)->toLoadItemsViaRequests([ 'https://example.com/users' => 'lengthAware/page1.json', ]); })->throws(InvalidPageInPathException::class, 'The pattern [/(\d+)(?!.*\d)/] could not capture any page from the path [/users].'); diff --git a/tests/Pest.php b/tests/Pest.php index 09d2ddd..5208efa 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -30,7 +30,7 @@ | */ -expect()->extend('toLoadItemsViaRequests', function (array $items, array $requests) { +expect()->extend('toLoadItemsViaRequests', function (array $requests) { $responses = $transactions = $expectedUris = []; foreach ($requests as $uri => $fixture) { @@ -44,7 +44,7 @@ Client::configure(['handler' => $stack]); - $this->sequence(...$items); + $this->sequence(...require fixture('items.php')); $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); From b15c6b619de76f48f28396632b9afeaf80228a1d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 30 Jan 2024 17:46:46 +1000 Subject: [PATCH 057/109] Declare strict types --- src/Concerns/RetriesHttpRequests.php | 2 ++ src/Concerns/SendsAsyncRequests.php | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php index fc73b5a..bbba639 100644 --- a/src/Concerns/RetriesHttpRequests.php +++ b/src/Concerns/RetriesHttpRequests.php @@ -1,5 +1,7 @@ Date: Tue, 30 Jan 2024 17:47:14 +1000 Subject: [PATCH 058/109] Reset the client on error --- tests/Pest.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/Pest.php b/tests/Pest.php index 5208efa..8a66ff7 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -44,7 +44,11 @@ Client::configure(['handler' => $stack]); - $this->sequence(...require fixture('items.php')); + try { + $this->sequence(...require fixture('items.php')); + } finally { + Client::reset(); + } $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); From d78b695b5517980af9586e2cbafb3a1ce92d4a64 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 30 Jan 2024 17:49:13 +1000 Subject: [PATCH 059/109] Support offset --- README.md | 37 +++--------------- src/Concerns/ResolvesPages.php | 8 +++- src/Concerns/YieldsPaginatedItems.php | 6 +++ src/Dtos/Config.php | 1 + src/LazyJsonPages.php | 10 +++++ src/Paginations/LengthAwarePagination.php | 20 ++++------ src/Paginations/Pagination.php | 5 +++ src/Paginations/TotalItemsAwarePagination.php | 10 +---- tests/Feature/StructureTest.php | 39 +++++++++++++++++++ 9 files changed, 83 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 636fe64..ba83c47 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,6 @@ The variable `$source` in our examples represents any [source](#-sources) that p ```php $lazyCollection = LazyJsonPages::from($source) ->totalItems('pagination.total_items') - ->perPage(20) ->offset() ->collect('results.*'); ``` @@ -157,44 +156,18 @@ LazyJsonPages::from($source)->lastPage('X-Last-Page'); APIs can expose their length information in the form of numbers (`total_pages: 10`) or URIs (`last_page: "https://example.com?page=10"`), Lazy JSON Pages supports both. -When dealing with a lot of data, it may be a good idea to fetch only 1 item on the first page and leverage the length information on that page to calculate the total number of pages/items without having to load all the other items of that page. - -We can do that by calling `perPage()` with: -- the number of items that we want to show per page (we can also override the pagination default) -- the query parameter or header that holds the number of items per page - -```php -// indicate that the number of items per page is defined by the `limit` query parameter, e.g. ?limit=50 -LazyJsonPages::from($source) - ->totalItems('pagination.total_items') - ->perPage(30, 'limit'); - -// indicate that the number of items per page is defined by the `X-Limit` header -LazyJsonPages::from($source) - ->totalItems('pagination.total_items') - ->perPage(30, header: 'X-Limit'); -``` - -Some APIs may not allow to request only 1 item per page, in these cases we can specify how many items should be loaded on the first page as third argument: +If the pagination works with an offset, we can configure it with the `offset()` method. The value of the offset will be calculated based on the number of items present on the first page: ```php +// indicate that the offset is defined by the `offset` query parameter, e.g. ?offset=50 LazyJsonPages::from($source) ->totalItems('pagination.total_items') - ->perPage(30, 'limit', 5); + ->offset(); +// indicate that the offset is defined by the `skip` query parameter, e.g. ?skip=50 LazyJsonPages::from($source) ->totalItems('pagination.total_items') - ->perPage(30, header: 'X-Limit', firstPageItems: 5); -``` - -We can leverage the `perPage()` strategy with all the length-aware methods seen before: - -```php -LazyJsonPages::from($source)->totalPages('pagination.total_pages')->perPage(30, 'limit'); - -LazyJsonPages::from($source)->totalItems('pagination.total_items')->perPage(30, 'limit'); - -LazyJsonPages::from($source)->lastPage('pagination.last_page')->perPage(30, 'limit'); + ->offset('skip'); ``` diff --git a/src/Concerns/ResolvesPages.php b/src/Concerns/ResolvesPages.php index 4b92eb2..53413f2 100644 --- a/src/Concerns/ResolvesPages.php +++ b/src/Concerns/ResolvesPages.php @@ -1,5 +1,7 @@ config->offsetKey) { + return Uri::withQueryValue($uri, $key, strval(($page - $this->config->firstPage) * $this->itemsPerPage)); + } + if (!$pattern = $this->config->pageInPath) { return Uri::withQueryValue($uri, $this->config->pageName, $page); } @@ -57,6 +63,6 @@ protected function uriForPage(UriInterface $uri, string $page): UriInterface throw new InvalidPageInPathException($path, $pattern); } - return $uri->withPath(substr_replace($path, $page, $matches[1][1], strlen($matches[1][0]))); + return $uri->withPath(substr_replace($path, $page, (int) $matches[1][1], strlen($matches[1][0]))); } } diff --git a/src/Concerns/YieldsPaginatedItems.php b/src/Concerns/YieldsPaginatedItems.php index 40f4364..0ec50ee 100644 --- a/src/Concerns/YieldsPaginatedItems.php +++ b/src/Concerns/YieldsPaginatedItems.php @@ -1,5 +1,7 @@ config->pointer]; if (($value = $response->getHeaderLine($key)) === '') { @@ -30,9 +33,12 @@ protected function yieldItemsAndReturnKey(ResponseInterface $response, string $k $value = $item->value; } else { yield $item; + ++$itemsPerPage; } } + $this->itemsPerPage ??= $itemsPerPage; + return $value; } diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 9f085ca..9aab571 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -21,6 +21,7 @@ public function __construct( public readonly ?Closure $nextPage = null, public readonly ?string $nextPageKey = null, public readonly ?int $lastPage = null, + public readonly ?string $offsetKey = null, public readonly int $async = 3, public readonly int $attempts = 3, public readonly ?Closure $backoff = null, diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 9e34a1d..0e543d7 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -153,6 +153,16 @@ public function lastPage(Closure|string $key): self return $this; } + /** + * Set the offset. + */ + public function offset(string $key = 'offset'): self + { + $this->config['offsetKey'] = $key; + + return $this; + } + /** * Fetch pages synchronously. */ diff --git a/src/Paginations/LengthAwarePagination.php b/src/Paginations/LengthAwarePagination.php index adc5c0c..8709bb9 100644 --- a/src/Paginations/LengthAwarePagination.php +++ b/src/Paginations/LengthAwarePagination.php @@ -9,7 +9,6 @@ use Closure; use Generator; use Illuminate\Support\LazyCollection; -use Psr\Http\Message\UriInterface; /** * The abstract implementation of a pagination that is aware of its length. @@ -44,11 +43,10 @@ protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): G * * @return Generator */ - protected function yieldItemsUntilPage(int $page, ?UriInterface $uri = null): Generator + protected function yieldItemsUntilPage(int $page): Generator { - $uri ??= $this->source->request()->getUri(); - $firstPageAlreadyFetched = strval($uri) == strval($this->source->request()->getUri()); - $chunkedPages = $this->chunkPages($page, $firstPageAlreadyFetched); + $uri = $this->source->request()->getUri(); + $chunkedPages = $this->chunkPages($page); foreach ($this->fetchPagesAsynchronously($chunkedPages, $uri) as $page) { yield from $this->yieldItemsFrom($page); @@ -60,15 +58,13 @@ protected function yieldItemsUntilPage(int $page, ?UriInterface $uri = null): Ge * * @return LazyCollection> */ - protected function chunkPages(int $pages, bool $shouldSkipFirstPage): LazyCollection + protected function chunkPages(int $pages): LazyCollection { - if ($pages == 0 || ($pages == 1 && $shouldSkipFirstPage)) { - return LazyCollection::empty(); - } - - $firstPage = $shouldSkipFirstPage ? $this->config->firstPage + 1 : $this->config->firstPage; + $firstPage = $this->config->firstPage + 1; $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; - return LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); + return $firstPage > $lastPage + ? LazyCollection::empty() + : LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); } } diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index c53c778..9de6856 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -27,6 +27,11 @@ abstract class Pagination implements IteratorAggregate */ public readonly Book $book; + /** + * The number of items per page. + */ + protected readonly int $itemsPerPage; + /** * Determine whether the configuration matches this pagination. */ diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 025e7fd..04977e6 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -29,19 +29,13 @@ public function matches(): bool */ public function getIterator(): Traversable { - $perPage = 0; - $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->totalItemsKey); - - foreach ($generator as $item) { - yield $item; - ++$perPage; - } + yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->totalItemsKey); if (!is_numeric($totalItems = $generator->getReturn())) { throw new InvalidKeyException($this->config->totalItemsKey); } - $totalPages = $perPage > 0 ? (int) ceil(intval($totalItems) / $perPage) : 0; + $totalPages = $this->itemsPerPage > 0 ? (int) ceil(intval($totalItems) / $this->itemsPerPage) : 0; yield from $this->yieldItemsUntilPage($totalPages); } diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index 22d4d75..440d18b 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -39,3 +39,42 @@ 'https://example.com/users' => 'lengthAware/page1.json', ]); })->throws(InvalidPageInPathException::class, 'The pattern [/(\d+)(?!.*\d)/] could not capture any page from the path [/users].'); + +it('supports paginations with offset', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->offset() + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?offset=5' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?offset=10' => 'lengthAware/page3.json', + ]); +}); + +it('supports paginations with limit and offset', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users?limit=5') + ->offset() + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users?limit=5' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?limit=5&offset=5' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?limit=5&offset=10' => 'lengthAware/page3.json', + ]); +}); + +it('supports paginations with custom offset', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->offset('skip') + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?skip=5' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?skip=10' => 'lengthAware/page3.json', + ]); +}); From 51b81c081674f7f8ac9fb7485c04505beef8afec Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 30 Jan 2024 19:43:42 +1000 Subject: [PATCH 060/109] Add docblock --- src/Sources/Source.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Sources/Source.php b/src/Sources/Source.php index c69d95d..b337f48 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -24,6 +24,9 @@ abstract public function request(): RequestInterface; */ abstract public function response(): ResponseInterface; + /** + * Instantiate the class. + */ final public function __construct( protected readonly mixed $source, ) {} From 209c060e78b4862faa52e3dc165b7a17405e6434 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 30 Jan 2024 19:49:44 +1000 Subject: [PATCH 061/109] Support custom pagination --- README.md | 35 ++++++++++++++++++ src/Dtos/Config.php | 10 ++++++ src/Exceptions/InvalidPaginationException.php | 23 ++++++++++++ src/LazyJsonPages.php | 10 ++++++ src/Paginations/AnyPagination.php | 12 +------ src/Paginations/CustomPagination.php | 36 +++++++++++++++++++ src/Paginations/Pagination.php | 16 ++++++--- tests/Feature/PaginationTest.php | 25 +++++++++++++ 8 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 src/Exceptions/InvalidPaginationException.php create mode 100644 src/Paginations/CustomPagination.php diff --git a/README.md b/README.md index ba83c47..56f63aa 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ composer require cerbero/lazy-json-pages * [๐Ÿ›๏ธ Pagination structure](#%EF%B8%8F-pagination-structure) * [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) * [โ†ช๏ธ Cursor and next-page paginations](#%EF%B8%8F-cursor-and-next-page-paginations) +* [๐Ÿ‘ฝ Custom pagination](#-custom-pagination) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) @@ -177,6 +178,40 @@ LazyJsonPages::from($source) > The documentation of this feature is a work in progress. +### ๐Ÿ‘ฝ Custom pagination + +Lazy JSON Pages provides several methods to extract items from the most popular pagination mechanisms. However if we need a custom solution, we can implement our own pagination. + +To implement a custom pagination, we need to extend `Pagination` and implement 1 method: + +```php +use Cerbero\LazyJsonPages\Paginations\Pagination; +use Traversable; + +class CustomPagination extends Pagination +{ + public function getIterator(): Traversable + { + // return a Traversable holding the paginated items + } +} +``` + +The parent class `Pagination` gives us access to 2 properties: +- `$source`: the mean pointing to the paginated JSON API (see [sources](#-sources)) +- `$config`: the configuration that we generated by chaining methods like `totalPages()` + +The method `getIterator()` defines the logic to extract paginated items in a memory-efficient way. Please refer to the [already existing paginations](https://github.com/cerbero90/json-parser/tree/master/src/Paginations) to see some implementations. + +Once the custom pagination is implemented, we can instruct Lazy JSON Pages to use it: + +```php +LazyJsonPages::from($source)->pagination(CustomPagination::class); +``` + +If you find yourself implementing the same custom pagination in different projects, feel free to send a PR and we will consider to support your custom pagination by default. Thank you in advance for any contribution! + + ### ๐Ÿš€ Requests optimization > [!WARNING] diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 9aab571..29244b8 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -4,10 +4,19 @@ namespace Cerbero\LazyJsonPages\Dtos; +use Cerbero\LazyJsonPages\Paginations\Pagination; use Closure; +/** + * The configuration + * + * @property-read class-string $pagination + */ final class Config { + /** + * Instantiate the class. + */ public function __construct( public readonly string $pointer, public readonly string $pageName = 'page', @@ -22,6 +31,7 @@ public function __construct( public readonly ?string $nextPageKey = null, public readonly ?int $lastPage = null, public readonly ?string $offsetKey = null, + public readonly ?string $pagination = null, public readonly int $async = 3, public readonly int $attempts = 3, public readonly ?Closure $backoff = null, diff --git a/src/Exceptions/InvalidPaginationException.php b/src/Exceptions/InvalidPaginationException.php new file mode 100644 index 0000000..97f4012 --- /dev/null +++ b/src/Exceptions/InvalidPaginationException.php @@ -0,0 +1,23 @@ +config['pagination'] = $class; + + return $this; + } + /** * Fetch pages synchronously. */ diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 2c01700..dd56b4e 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -19,23 +19,13 @@ class AnyPagination extends Pagination */ protected array $supportedPaginations = [ // CursorPagination::class, - // CustomPagination::class, + CustomPagination::class, // LastPageAwarePagination::class, - // LimitPagination::class, // LinkHeaderPagination::class, - // OffsetPagination::class, TotalItemsAwarePagination::class, TotalPagesAwarePagination::class, ]; - /** - * Determine whether this pagination matches the configuration. - */ - public function matches(): bool - { - return true; - } - /** * Yield the paginated items. * diff --git a/src/Paginations/CustomPagination.php b/src/Paginations/CustomPagination.php new file mode 100644 index 0000000..18339f2 --- /dev/null +++ b/src/Paginations/CustomPagination.php @@ -0,0 +1,36 @@ +config->pagination !== null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + if (!is_subclass_of($this->config->pagination, Pagination::class)) { + throw new InvalidPaginationException($this->config->pagination); + } + + yield from new $this->config->pagination($this->source, $this->config); + } +} diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 9de6856..c9cfd80 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -32,11 +32,6 @@ abstract class Pagination implements IteratorAggregate */ protected readonly int $itemsPerPage; - /** - * Determine whether the configuration matches this pagination. - */ - abstract public function matches(): bool; - /** * Yield the paginated items. * @@ -44,10 +39,21 @@ abstract public function matches(): bool; */ abstract public function getIterator(): Traversable; + /** + * Instantiate the class. + */ final public function __construct( protected readonly Source $source, protected readonly Config $config, ) { $this->book = new Book(); } + + /** + * Determine whether the configuration matches this pagination. + */ + public function matches(): bool + { + return true; + } } diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index 7013a45..1ee9e58 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -1,6 +1,8 @@ 'lengthAware/page3.json', ]); }); + +it('supports custom paginations', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->pagination(TotalPagesAwarePagination::class) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', + 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + ]); +}); + +it('fails if an invalid custom pagination is provided', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->pagination('Invalid') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + ]); +})->throws(InvalidPaginationException::class, 'The class [Invalid] should extend [Cerbero\LazyJsonPages\Paginations\Pagination].'); From b3b6e80d8aa1a4fc54256fab860ab5b92aa5c476 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 31 Jan 2024 20:35:40 +1000 Subject: [PATCH 062/109] Update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 56f63aa..fa065fb 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,10 @@ If the length information is nested in the JSON body, we can use dot-notation to Otherwise, if the length information is displayed in the headers, we can use the same methods to gather it by simply defining the name of the header: ```php +LazyJsonPages::from($source)->totalPages('X-Total-Pages'); + +LazyJsonPages::from($source)->totalItems('X-Total-Items'); + LazyJsonPages::from($source)->lastPage('X-Last-Page'); ``` From 74f9a89b00094d474ed5e3faa5e9d745e856394e Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 31 Jan 2024 20:37:15 +1000 Subject: [PATCH 063/109] Improve matching logic --- src/Paginations/TotalPagesAwarePagination.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index b8f9ba3..c2aeabe 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -16,8 +16,7 @@ class TotalPagesAwarePagination extends LengthAwarePagination */ public function matches(): bool { - return $this->config->totalPagesKey !== null - && $this->config->perPage === null; + return $this->config->totalPagesKey !== null; } /** From c3d9329564c86dad8db2339b520c9a4c461745f5 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 31 Jan 2024 20:38:39 +1000 Subject: [PATCH 064/109] Enable headers testing --- tests/Feature/Datasets.php | 13 +++++++++++++ tests/Pest.php | 7 ++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index e9b15e7..83cddd1 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -1,5 +1,7 @@ [$psr7Request, true]; yield 'Symfony request' => [SymfonyRequest::create($uri), true]; }); + +dataset('length-aware', function () { + yield 'total pages aware' => fn(LazyJsonPages $instance) => $instance->totalPages('meta.total_pages'); + yield 'total items aware' => fn(LazyJsonPages $instance) => $instance->totalItems('meta.total_items'); + yield 'last page aware' => fn(LazyJsonPages $instance) => $instance->lastPage('meta.last_page'); + yield 'custom pagination' => fn(LazyJsonPages $instance) => $instance->pagination(TotalPagesAwarePagination::class)->totalPages('meta.total_pages'); + yield 'total pages aware by header' => fn(LazyJsonPages $instance) => $instance->totalPages('X-Total-Pages'); + yield 'total items aware by header' => fn(LazyJsonPages $instance) => $instance->totalItems('X-Total-Items'); + yield 'last page aware by header' => fn(LazyJsonPages $instance) => $instance->lastPage('X-Last-Page'); + yield 'custom pagination by header' => fn(LazyJsonPages $instance) => $instance->pagination(TotalPagesAwarePagination::class)->totalPages('X-Total-Pages'); +}); diff --git a/tests/Pest.php b/tests/Pest.php index 8a66ff7..5bd2ba4 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -32,9 +32,14 @@ expect()->extend('toLoadItemsViaRequests', function (array $requests) { $responses = $transactions = $expectedUris = []; + $headers = [ + 'X-Total-Pages' => 3, + 'X-Total-Items' => 14, + ]; foreach ($requests as $uri => $fixture) { - $responses[] = new Response(body: file_get_contents(fixture($fixture))); + $headers['X-Last-Page'] ??= str_contains($fixture, 'page0') ? 2 : 3; + $responses[] = new Response(body: file_get_contents(fixture($fixture)), headers: $headers); $expectedUris[] = $uri; } From f486a69a95cd1bad1a4a34c7ae7218aff4477257 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 31 Jan 2024 20:39:44 +1000 Subject: [PATCH 065/109] Improve tests with dataset and cases with 0 as first page --- tests/Feature/PaginationTest.php | 35 ++++++------------- tests/Feature/StructureTest.php | 14 ++++++++ tests/fixtures/lengthAware/page1.json | 3 +- tests/fixtures/lengthAware/page2.json | 3 +- tests/fixtures/lengthAware/page3.json | 3 +- .../fixtures/lengthAwareFirstPage0/page0.json | 24 +++++++++++++ .../fixtures/lengthAwareFirstPage0/page1.json | 24 +++++++++++++ .../fixtures/lengthAwareFirstPage0/page2.json | 21 +++++++++++ 8 files changed, 99 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/lengthAwareFirstPage0/page0.json create mode 100644 tests/fixtures/lengthAwareFirstPage0/page1.json create mode 100644 tests/fixtures/lengthAwareFirstPage0/page2.json diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index 1ee9e58..662141a 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -2,23 +2,9 @@ use Cerbero\LazyJsonPages\Exceptions\InvalidPaginationException; use Cerbero\LazyJsonPages\LazyJsonPages; -use Cerbero\LazyJsonPages\Paginations\TotalPagesAwarePagination; -it('supports paginations aware of their total pages', function () { - $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') - ->totalPages('meta.total_pages') - ->collect('data.*'); - - expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', - ]); -}); - -it('supports paginations aware of their total items', function () { - $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') - ->totalItems('meta.total_items') +it('supports length-aware paginations', function (Closure $configure) { + $lazyCollection = $configure(LazyJsonPages::from('https://example.com/api/v1/users')) ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ @@ -26,20 +12,19 @@ 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', ]); -}); +})->with('length-aware'); -it('supports custom paginations', function () { - $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') - ->pagination(TotalPagesAwarePagination::class) - ->totalPages('meta.total_pages') +it('supports length-aware paginations having 0 as first page', function (Closure $configure) { + $lazyCollection = $configure(LazyJsonPages::from('https://example.com/api/v1/users')) + ->firstPage(0) ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users' => 'lengthAwareFirstPage0/page0.json', + 'https://example.com/api/v1/users?page=1' => 'lengthAwareFirstPage0/page1.json', + 'https://example.com/api/v1/users?page=2' => 'lengthAwareFirstPage0/page2.json', ]); -}); +})->with('length-aware'); it('fails if an invalid custom pagination is provided', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index 440d18b..8acb1b3 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -53,6 +53,20 @@ ]); }); +it('supports paginations with offset and 0 as first page', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->offset() + ->firstPage(0) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'lengthAwareFirstPage0/page0.json', + 'https://example.com/api/v1/users?offset=5' => 'lengthAwareFirstPage0/page1.json', + 'https://example.com/api/v1/users?offset=10' => 'lengthAwareFirstPage0/page2.json', + ]); +}); + it('supports paginations with limit and offset', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users?limit=5') ->offset() diff --git a/tests/fixtures/lengthAware/page1.json b/tests/fixtures/lengthAware/page1.json index 55c3fb4..5fa0fe8 100644 --- a/tests/fixtures/lengthAware/page1.json +++ b/tests/fixtures/lengthAware/page1.json @@ -18,6 +18,7 @@ ], "meta": { "total_pages": 3, - "total_items": 14 + "total_items": 14, + "last_page": 3 } } diff --git a/tests/fixtures/lengthAware/page2.json b/tests/fixtures/lengthAware/page2.json index 0f13b6b..9765209 100644 --- a/tests/fixtures/lengthAware/page2.json +++ b/tests/fixtures/lengthAware/page2.json @@ -18,6 +18,7 @@ ], "meta": { "total_pages": 3, - "total_items": 14 + "total_items": 14, + "last_page": 3 } } diff --git a/tests/fixtures/lengthAware/page3.json b/tests/fixtures/lengthAware/page3.json index 9199d0a..de48011 100644 --- a/tests/fixtures/lengthAware/page3.json +++ b/tests/fixtures/lengthAware/page3.json @@ -15,6 +15,7 @@ ], "meta": { "total_pages": 3, - "total_items": 14 + "total_items": 14, + "last_page": 3 } } diff --git a/tests/fixtures/lengthAwareFirstPage0/page0.json b/tests/fixtures/lengthAwareFirstPage0/page0.json new file mode 100644 index 0000000..65197c5 --- /dev/null +++ b/tests/fixtures/lengthAwareFirstPage0/page0.json @@ -0,0 +1,24 @@ +{ + "data": [ + { + "name": "item1" + }, + { + "name": "item2" + }, + { + "name": "item3" + }, + { + "name": "item4" + }, + { + "name": "item5" + } + ], + "meta": { + "total_pages": 3, + "total_items": 14, + "last_page": 2 + } +} diff --git a/tests/fixtures/lengthAwareFirstPage0/page1.json b/tests/fixtures/lengthAwareFirstPage0/page1.json new file mode 100644 index 0000000..5b3bc68 --- /dev/null +++ b/tests/fixtures/lengthAwareFirstPage0/page1.json @@ -0,0 +1,24 @@ +{ + "data": [ + { + "name": "item6" + }, + { + "name": "item7" + }, + { + "name": "item8" + }, + { + "name": "item9" + }, + { + "name": "item10" + } + ], + "meta": { + "total_pages": 3, + "total_items": 14, + "last_page": 2 + } +} diff --git a/tests/fixtures/lengthAwareFirstPage0/page2.json b/tests/fixtures/lengthAwareFirstPage0/page2.json new file mode 100644 index 0000000..129c482 --- /dev/null +++ b/tests/fixtures/lengthAwareFirstPage0/page2.json @@ -0,0 +1,21 @@ +{ + "data": [ + { + "name": "item11" + }, + { + "name": "item12" + }, + { + "name": "item13" + }, + { + "name": "item14" + } + ], + "meta": { + "total_pages": 3, + "total_items": 14, + "last_page": 2 + } +} From 36a7faa1fe8a4a4cc8f75b549eccc3abc8990e42 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 31 Jan 2024 20:40:22 +1000 Subject: [PATCH 066/109] Support paginations aware of their last page --- src/Dtos/Config.php | 2 +- src/LazyJsonPages.php | 4 +-- src/Paginations/AnyPagination.php | 2 +- src/Paginations/LastPageAwarePagination.php | 33 +++++++++++++++++++ src/Paginations/TotalItemsAwarePagination.php | 2 +- 5 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 src/Paginations/LastPageAwarePagination.php diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 29244b8..667d8ef 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -29,7 +29,7 @@ public function __construct( public readonly ?int $perPageOverride = null, public readonly ?Closure $nextPage = null, public readonly ?string $nextPageKey = null, - public readonly ?int $lastPage = null, + public readonly ?string $lastPageKey = null, public readonly ?string $offsetKey = null, public readonly ?string $pagination = null, public readonly int $async = 3, diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index c7307e7..df99caf 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -146,9 +146,9 @@ public function nextPage(Closure|string $key): self /** * Set the number of the last page. */ - public function lastPage(Closure|string $key): self + public function lastPage(string $key): self { - $this->config['lastPage'] = $this->integerFromResponse($key); + $this->config['lastPageKey'] = $key; return $this; } diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index dd56b4e..1af0cbe 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -20,7 +20,7 @@ class AnyPagination extends Pagination protected array $supportedPaginations = [ // CursorPagination::class, CustomPagination::class, - // LastPageAwarePagination::class, + LastPageAwarePagination::class, // LinkHeaderPagination::class, TotalItemsAwarePagination::class, TotalPagesAwarePagination::class, diff --git a/src/Paginations/LastPageAwarePagination.php b/src/Paginations/LastPageAwarePagination.php new file mode 100644 index 0000000..a764f23 --- /dev/null +++ b/src/Paginations/LastPageAwarePagination.php @@ -0,0 +1,33 @@ +config->lastPageKey !== null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function (int $page) { + return $this->config->firstPage === 0 ? $page + 1 : $page; + }); + } +} diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 04977e6..21087c8 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -19,7 +19,7 @@ public function matches(): bool { return $this->config->totalItemsKey !== null && $this->config->totalPagesKey === null - && $this->config->perPage === null; + && $this->config->lastPageKey === null; } /** From e2c80b0f193d812d35c3741a0b98b7fc3c98cc78 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 16:32:51 +1000 Subject: [PATCH 067/109] Move fixtures --- tests/Feature/Datasets.php | 2 +- tests/Feature/PaginationTest.php | 26 +++++++++---- tests/Feature/SourceTest.php | 6 +-- tests/Feature/StructureTest.php | 38 +++++++++---------- tests/Sources/CustomSourceSample.php | 2 +- .../{lengthAware => pagination}/page1.json | 3 +- .../{lengthAware => pagination}/page2.json | 3 +- .../{lengthAware => pagination}/page3.json | 3 +- .../page0.json | 0 .../page1.json | 0 .../page2.json | 0 11 files changed, 49 insertions(+), 34 deletions(-) rename tests/fixtures/{lengthAware => pagination}/page1.json (86%) rename tests/fixtures/{lengthAware => pagination}/page2.json (86%) rename tests/fixtures/{lengthAware => pagination}/page3.json (86%) rename tests/fixtures/{lengthAwareFirstPage0 => paginationFirstPage0}/page0.json (100%) rename tests/fixtures/{lengthAwareFirstPage0 => paginationFirstPage0}/page1.json (100%) rename tests/fixtures/{lengthAwareFirstPage0 => paginationFirstPage0}/page2.json (100%) diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index 83cddd1..369b435 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -14,7 +14,7 @@ dataset('sources', function () { $uri = 'https://example.com/api/v1/users'; $psr7Request = new Psr7Request('GET', $uri); - $psr7Response = new Psr7Response(body: file_get_contents(fixture('lengthAware/page1.json'))); + $psr7Response = new Psr7Response(body: file_get_contents(fixture('pagination/page1.json'))); $laravelClientResponse = new LaravelClientResponse($psr7Response); $laravelClientResponse->transferStats = new TransferStats($psr7Request, $psr7Response); diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index 662141a..aa66ff8 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -8,9 +8,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', ]); })->with('length-aware'); @@ -20,18 +20,30 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAwareFirstPage0/page0.json', - 'https://example.com/api/v1/users?page=1' => 'lengthAwareFirstPage0/page1.json', - 'https://example.com/api/v1/users?page=2' => 'lengthAwareFirstPage0/page2.json', + 'https://example.com/api/v1/users' => 'paginationFirstPage0/page0.json', + 'https://example.com/api/v1/users?page=1' => 'paginationFirstPage0/page1.json', + 'https://example.com/api/v1/users?page=2' => 'paginationFirstPage0/page2.json', ]); })->with('length-aware'); +it('supports cursor-aware paginations', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->cursor('meta.cursor') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=cursor1' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=cursor2' => 'pagination/page3.json', + ]); +}); + it('fails if an invalid custom pagination is provided', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->pagination('Invalid') ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', + 'https://example.com/api/v1/users' => 'pagination/page1.json', ]); })->throws(InvalidPaginationException::class, 'The class [Invalid] should extend [Cerbero\LazyJsonPages\Paginations\Pagination].'); diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php index f68374f..09a7390 100644 --- a/tests/Feature/SourceTest.php +++ b/tests/Feature/SourceTest.php @@ -8,8 +8,8 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'lengthAware/page1.json'] : [], - 'https://example.com/api/v1/users?page=2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?page=3' => 'lengthAware/page3.json', + ...$requestsFirstPage ? ['https://example.com/api/v1/users' => 'pagination/page1.json'] : [], + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', ]); })->with('sources'); diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index 8acb1b3..c269d39 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -10,9 +10,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users/page/1' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users/page/2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users/page/3' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users/page/1' => 'pagination/page1.json', + 'https://example.com/api/v1/users/page/2' => 'pagination/page2.json', + 'https://example.com/api/v1/users/page/3' => 'pagination/page3.json', ]); }); @@ -23,9 +23,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users/page1' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users/page2' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users/page3' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users/page1' => 'pagination/page1.json', + 'https://example.com/api/v1/users/page2' => 'pagination/page2.json', + 'https://example.com/api/v1/users/page3' => 'pagination/page3.json', ]); }); @@ -36,7 +36,7 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/users' => 'lengthAware/page1.json', + 'https://example.com/users' => 'pagination/page1.json', ]); })->throws(InvalidPageInPathException::class, 'The pattern [/(\d+)(?!.*\d)/] could not capture any page from the path [/users].'); @@ -47,9 +47,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?offset=5' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?offset=10' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?offset=5' => 'pagination/page2.json', + 'https://example.com/api/v1/users?offset=10' => 'pagination/page3.json', ]); }); @@ -61,9 +61,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAwareFirstPage0/page0.json', - 'https://example.com/api/v1/users?offset=5' => 'lengthAwareFirstPage0/page1.json', - 'https://example.com/api/v1/users?offset=10' => 'lengthAwareFirstPage0/page2.json', + 'https://example.com/api/v1/users' => 'paginationFirstPage0/page0.json', + 'https://example.com/api/v1/users?offset=5' => 'paginationFirstPage0/page1.json', + 'https://example.com/api/v1/users?offset=10' => 'paginationFirstPage0/page2.json', ]); }); @@ -74,9 +74,9 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users?limit=5' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?limit=5&offset=5' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?limit=5&offset=10' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users?limit=5' => 'pagination/page1.json', + 'https://example.com/api/v1/users?limit=5&offset=5' => 'pagination/page2.json', + 'https://example.com/api/v1/users?limit=5&offset=10' => 'pagination/page3.json', ]); }); @@ -87,8 +87,8 @@ ->collect('data.*'); expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'lengthAware/page1.json', - 'https://example.com/api/v1/users?skip=5' => 'lengthAware/page2.json', - 'https://example.com/api/v1/users?skip=10' => 'lengthAware/page3.json', + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?skip=5' => 'pagination/page2.json', + 'https://example.com/api/v1/users?skip=10' => 'pagination/page3.json', ]); }); diff --git a/tests/Sources/CustomSourceSample.php b/tests/Sources/CustomSourceSample.php index 1cc4d37..a7cabde 100644 --- a/tests/Sources/CustomSourceSample.php +++ b/tests/Sources/CustomSourceSample.php @@ -29,6 +29,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return new Response(body: file_get_contents(fixture('lengthAware/page1.json'))); + return new Response(body: file_get_contents(fixture('pagination/page1.json'))); } } diff --git a/tests/fixtures/lengthAware/page1.json b/tests/fixtures/pagination/page1.json similarity index 86% rename from tests/fixtures/lengthAware/page1.json rename to tests/fixtures/pagination/page1.json index 5fa0fe8..d483994 100644 --- a/tests/fixtures/lengthAware/page1.json +++ b/tests/fixtures/pagination/page1.json @@ -19,6 +19,7 @@ "meta": { "total_pages": 3, "total_items": 14, - "last_page": 3 + "last_page": 3, + "cursor": "cursor1" } } diff --git a/tests/fixtures/lengthAware/page2.json b/tests/fixtures/pagination/page2.json similarity index 86% rename from tests/fixtures/lengthAware/page2.json rename to tests/fixtures/pagination/page2.json index 9765209..f592a12 100644 --- a/tests/fixtures/lengthAware/page2.json +++ b/tests/fixtures/pagination/page2.json @@ -19,6 +19,7 @@ "meta": { "total_pages": 3, "total_items": 14, - "last_page": 3 + "last_page": 3, + "cursor": "cursor2" } } diff --git a/tests/fixtures/lengthAware/page3.json b/tests/fixtures/pagination/page3.json similarity index 86% rename from tests/fixtures/lengthAware/page3.json rename to tests/fixtures/pagination/page3.json index de48011..ef375ac 100644 --- a/tests/fixtures/lengthAware/page3.json +++ b/tests/fixtures/pagination/page3.json @@ -16,6 +16,7 @@ "meta": { "total_pages": 3, "total_items": 14, - "last_page": 3 + "last_page": 3, + "cursor": null } } diff --git a/tests/fixtures/lengthAwareFirstPage0/page0.json b/tests/fixtures/paginationFirstPage0/page0.json similarity index 100% rename from tests/fixtures/lengthAwareFirstPage0/page0.json rename to tests/fixtures/paginationFirstPage0/page0.json diff --git a/tests/fixtures/lengthAwareFirstPage0/page1.json b/tests/fixtures/paginationFirstPage0/page1.json similarity index 100% rename from tests/fixtures/lengthAwareFirstPage0/page1.json rename to tests/fixtures/paginationFirstPage0/page1.json diff --git a/tests/fixtures/lengthAwareFirstPage0/page2.json b/tests/fixtures/paginationFirstPage0/page2.json similarity index 100% rename from tests/fixtures/lengthAwareFirstPage0/page2.json rename to tests/fixtures/paginationFirstPage0/page2.json From 5f67547730daa19b976a68aa2977b3e1ce1bf72f Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 16:36:06 +1000 Subject: [PATCH 068/109] Client is automatically reset --- tests/Pest.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/Pest.php b/tests/Pest.php index 5bd2ba4..3f985fd 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -49,11 +49,7 @@ Client::configure(['handler' => $stack]); - try { - $this->sequence(...require fixture('items.php')); - } finally { - Client::reset(); - } + $this->sequence(...require fixture('items.php')); $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); From 4ee6700e184a9f7fef52ad4b979679e2e1bb5c6c Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 16:36:57 +1000 Subject: [PATCH 069/109] Rename pointer to itemsPointer --- src/Concerns/YieldsPaginatedItems.php | 4 ++-- src/Dtos/Config.php | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Concerns/YieldsPaginatedItems.php b/src/Concerns/YieldsPaginatedItems.php index 0ec50ee..26759dd 100644 --- a/src/Concerns/YieldsPaginatedItems.php +++ b/src/Concerns/YieldsPaginatedItems.php @@ -22,7 +22,7 @@ trait YieldsPaginatedItems protected function yieldItemsAndReturnKey(ResponseInterface $response, string $key): Generator { $itemsPerPage = 0; - $pointers = [$this->config->pointer]; + $pointers = [$this->config->itemsPointer]; if (($value = $response->getHeaderLine($key)) === '') { $pointers[DotsConverter::toPointer($key)] = fn(mixed $value) => (object) compact('value'); @@ -49,6 +49,6 @@ protected function yieldItemsAndReturnKey(ResponseInterface $response, string $k */ protected function yieldItemsFrom(mixed $source): Generator { - yield from JsonParser::parse($source)->pointer($this->config->pointer); + yield from JsonParser::parse($source)->pointer($this->config->itemsPointer); } } diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 667d8ef..3106e27 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -18,17 +18,13 @@ final class Config * Instantiate the class. */ public function __construct( - public readonly string $pointer, + public readonly string $itemsPointer, public readonly string $pageName = 'page', - public readonly ?string $pageInPath = null, public readonly int $firstPage = 1, + public readonly ?string $pageInPath = null, public readonly ?string $totalPagesKey = null, public readonly ?string $totalItemsKey = null, - public readonly ?int $perPage = null, - public readonly ?string $perPageKey = null, - public readonly ?int $perPageOverride = null, - public readonly ?Closure $nextPage = null, - public readonly ?string $nextPageKey = null, + public readonly ?string $cursorKey = null, public readonly ?string $lastPageKey = null, public readonly ?string $offsetKey = null, public readonly ?string $pagination = null, From 77b43d364faf1d55e7e4aeb96287452553c107eb Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 16:37:50 +1000 Subject: [PATCH 070/109] Fix page resolution for endpoints --- src/Concerns/ResolvesPages.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Concerns/ResolvesPages.php b/src/Concerns/ResolvesPages.php index 53413f2..e0d8dda 100644 --- a/src/Concerns/ResolvesPages.php +++ b/src/Concerns/ResolvesPages.php @@ -4,6 +4,7 @@ namespace Cerbero\LazyJsonPages\Concerns; +use Cerbero\JsonParser\Concerns\DetectsEndpoints; use Cerbero\LazyJsonPages\Exceptions\InvalidPageInPathException; use GuzzleHttp\Psr7\Uri; use Psr\Http\Message\UriInterface; @@ -13,6 +14,8 @@ */ trait ResolvesPages { + use DetectsEndpoints; + /** * Retrieve the page out of the given value. * @@ -23,8 +26,8 @@ protected function toPage(mixed $value, bool $onlyNumerics = true): string|int|n return match (true) { is_numeric($value) => (int) $value, !is_string($value) || $value === '' => null, - !$parsedUri = parse_url($value) => $onlyNumerics ? null : $value, - default => $this->pageFromParsedUri($parsedUri, $onlyNumerics), + !$this->isEndpoint($value) => $onlyNumerics ? null : $value, + default => $this->pageFromParsedUri(parse_url($value), $onlyNumerics), }; } From 0766e3c162b6de9f8812a8186593d3a627b41efe Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 16:38:03 +1000 Subject: [PATCH 071/109] Implement cursor pagination --- src/LazyJsonPages.php | 51 ++++++---------------------- src/Paginations/AnyPagination.php | 15 +++++++- src/Paginations/CursorPagination.php | 44 ++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 42 deletions(-) create mode 100644 src/Paginations/CursorPagination.php diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index df99caf..99fb3e7 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -95,22 +95,6 @@ public function totalPages(string $key): self return $this; } - /** - * Retrieve an integer from the response. - */ - private function integerFromResponse(Closure|string $key, int $minimum = 0): int - { - return (int) max($minimum, $this->valueFromResponse($key)); - } - - /** - * Retrieve a value from the response. - */ - private function valueFromResponse(Closure|string $key): mixed - { - return $key instanceof Closure ? $key($this->source->response()) : $this->source->response($key); - } - /** * Set the total number of items. */ @@ -122,23 +106,11 @@ public function totalItems(string $key): self } /** - * Set the number of items per page and optionally override it. + * Set the cursor or next page. */ - public function perPage(int $items, ?string $key = null, int $firstPageItems = 1): self + public function cursor(string $key): self { - $this->config['perPage'] = max(1, $key ? $firstPageItems : $items); - $this->config['perPageKey'] = $key; - $this->config['perPageOverride'] = $key ? max(1, $items) : null; - - return $this; - } - - /** - * Set the next page. - */ - public function nextPage(Closure|string $key): self - { - $this->config['nextPage'] = $this->valueFromResponse($key); + $this->config['cursorKey'] = $key; return $this; } @@ -236,23 +208,20 @@ public function backoff(Closure $callback): self * Retrieve a lazy collection yielding the paginated items. * * @return LazyCollection + * @throws UnsupportedPaginationException */ public function collect(string $dot = '*'): LazyCollection { - $this->config['pointer'] = DotsConverter::toPointer($dot); - Client::configure($this->requestOptions); - return new LazyCollection(function () { - $items = new AnyPagination($this->source, new Config(...$this->config)); + $config = new Config(...$this->config, itemsPointer: DotsConverter::toPointer($dot)); - // yield each item within a loop - instead of using `yield from` - to ignore the actual item index - // and ensure indexes continuity, otherwise the index of items always starts from 0 on every page. - foreach ($items as $item) { - yield $item; + return new LazyCollection(function () use ($config) { + try { + yield from new AnyPagination($this->source, $config); + } finally { + Client::reset(); } - - Client::reset(); }); } } diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 1af0cbe..b2987e2 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -18,7 +18,7 @@ class AnyPagination extends Pagination * @var class-string[] */ protected array $supportedPaginations = [ - // CursorPagination::class, + CursorPagination::class, CustomPagination::class, LastPageAwarePagination::class, // LinkHeaderPagination::class, @@ -30,8 +30,21 @@ class AnyPagination extends Pagination * Yield the paginated items. * * @return Traversable + * @throws UnsupportedPaginationException */ public function getIterator(): Traversable + { + // yield only items and not their related index to ensure indexes continuity + // otherwise the actual indexes always start from 0 on every page. + foreach ($this->matchingPagination() as $item) { + yield $item; + } + } + + /** + * Retrieve the pagination matching with the configuration. + */ + protected function matchingPagination(): Pagination { foreach ($this->supportedPaginations as $class) { $pagination = new $class($this->source, $this->config); diff --git a/src/Paginations/CursorPagination.php b/src/Paginations/CursorPagination.php new file mode 100644 index 0000000..ffa9812 --- /dev/null +++ b/src/Paginations/CursorPagination.php @@ -0,0 +1,44 @@ +config->cursorKey !== null + && $this->config->totalItemsKey === null + && $this->config->totalPagesKey === null + && $this->config->lastPageKey === null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + */ + public function getIterator(): Traversable + { + yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->cursorKey); + + $request = clone $this->source->request(); + + while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { + $uri = $this->uriForPage($request->getUri(), (string) $cursor); + $response = Client::instance()->send($request->withUri($uri)); + + yield from $generator = $this->yieldItemsAndReturnKey($response, $this->config->cursorKey); + } + } +} From 8816db414ed3630007419e97faa36bd24ff4589d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 17:06:30 +1000 Subject: [PATCH 072/109] Update readme --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fa065fb..6ad363d 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ composer require cerbero/lazy-json-pages * [๐Ÿ’ง Sources](#-sources) * [๐Ÿ›๏ธ Pagination structure](#%EF%B8%8F-pagination-structure) * [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) -* [โ†ช๏ธ Cursor and next-page paginations](#%EF%B8%8F-cursor-and-next-page-paginations) +* [โ†ช๏ธ Cursor-aware paginations](#%EF%B8%8F-cursor-and-next-page-paginations) * [๐Ÿ‘ฝ Custom pagination](#-custom-pagination) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) @@ -176,10 +176,17 @@ LazyJsonPages::from($source) ``` -### โ†ช๏ธ Cursor and next-page paginations +### โ†ช๏ธ Cursor-aware paginations -> [!WARNING] -> The documentation of this feature is a work in progress. +Not all paginations are [length-aware](#-length-aware-paginations), some may be built in a way where each page has a cursor pointing to the next page. + +We can tackle this kind of pagination by indicating the key or the header holding the cursor: + +```php +LazyJsonPages::from($source)->cursor('pagination.cursor'); +``` + +The cursor may be a number, a string or a URI: Lazy JSON Pages supports them all. ### ๐Ÿ‘ฝ Custom pagination From cd7116c83812d89d1e21231c0bd06d4bf8e604ac Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 17:07:14 +1000 Subject: [PATCH 073/109] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ad363d..8d46880 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ composer require cerbero/lazy-json-pages * [๐Ÿ’ง Sources](#-sources) * [๐Ÿ›๏ธ Pagination structure](#%EF%B8%8F-pagination-structure) * [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) -* [โ†ช๏ธ Cursor-aware paginations](#%EF%B8%8F-cursor-and-next-page-paginations) +* [โ†ช๏ธ Cursor-aware paginations](#%EF%B8%8F-cursor-aware-paginations) * [๐Ÿ‘ฝ Custom pagination](#-custom-pagination) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) From 90996e7b1b3755007229571e6d9755502c6b01fc Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Wed, 7 Feb 2024 17:09:58 +1000 Subject: [PATCH 074/109] Update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8d46880..898d9ef 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,8 @@ We can tackle this kind of pagination by indicating the key or the header holdin ```php LazyJsonPages::from($source)->cursor('pagination.cursor'); + +LazyJsonPages::from($source)->cursor('X-Cursor'); ``` The cursor may be a number, a string or a URI: Lazy JSON Pages supports them all. From a0923171cf373f3ccb63526825adc6262bd296dc Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 10 Feb 2024 20:53:45 +1000 Subject: [PATCH 075/109] Update readme --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 898d9ef..9b66239 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ composer require cerbero/lazy-json-pages * [๐Ÿ›๏ธ Pagination structure](#%EF%B8%8F-pagination-structure) * [๐Ÿ“ Length-aware paginations](#-length-aware-paginations) * [โ†ช๏ธ Cursor-aware paginations](#%EF%B8%8F-cursor-aware-paginations) -* [๐Ÿ‘ฝ Custom pagination](#-custom-pagination) +* [๐Ÿ”— Link header paginations](#-link-header-paginations) +* [๐Ÿ‘ฝ Custom paginations](#-custom-paginations) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) @@ -98,6 +99,41 @@ $source = new GuzzleHttp\Psr7\Request('GET', 'https://example.com/api'); $source = Http::withToken($bearer)->get('https://example.com/api'); ``` +If none of the above sources satifies our use case, we can implement our own source. + +To implement a custom source, we need to extend `Source` and implement 2 methods: + +```php +use Cerbero\LazyJsonPages\Sources\Source; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; + +class CustomSource extends Source +{ + public function request(): RequestInterface + { + // return a PSR-7 request + } + + public function response(): ResponseInterface + { + // return a PSR-7 response + } +} +``` + +The parent class `Source` gives us access to the property `$source`, which is the custom source for our use case. + +The methods to implement respectively turn our custom source into a PSR-7 request and a PSR-7 response. Please refer to the [already existing sources](https://github.com/cerbero90/json-parser/tree/master/src/Sources) to see some implementations. + +Once the custom source is implemented, we can instruct Lazy JSON Pages to use it: + +```php +LazyJsonPages::from(new CustomSource($source)); +``` + +If you find yourself implementing the same custom source in different projects, feel free to send a PR and we will consider to support your custom source by default. Thank you in advance for any contribution! + ### ๐Ÿ›๏ธ Pagination structure @@ -191,7 +227,23 @@ LazyJsonPages::from($source)->cursor('X-Cursor'); The cursor may be a number, a string or a URI: Lazy JSON Pages supports them all. -### ๐Ÿ‘ฝ Custom pagination +### ๐Ÿ”— Link header paginations + +Some paginated API responses include a header called `Link`. An example is [GitHub](https://api.github.com/repos/octocat/hello-world/issues?state=open): if we inspect the response headers, we can see the `Link` header looking like this: + +``` +; rel="next", +; rel="last" +``` + +To lazy-load the items of a Link header pagination, we can chain the method `linkHeader()`: + +```php +LazyJsonPages::from($source)->linkHeader(); +``` + + +### ๐Ÿ‘ฝ Custom paginations Lazy JSON Pages provides several methods to extract items from the most popular pagination mechanisms. However if we need a custom solution, we can implement our own pagination. @@ -205,13 +257,13 @@ class CustomPagination extends Pagination { public function getIterator(): Traversable { - // return a Traversable holding the paginated items + // return a Traversable yielding the paginated items } } ``` The parent class `Pagination` gives us access to 2 properties: -- `$source`: the mean pointing to the paginated JSON API (see [sources](#-sources)) +- `$source`: the [source](#-sources) pointing to the paginated JSON API - `$config`: the configuration that we generated by chaining methods like `totalPages()` The method `getIterator()` defines the logic to extract paginated items in a memory-efficient way. Please refer to the [already existing paginations](https://github.com/cerbero90/json-parser/tree/master/src/Paginations) to see some implementations. From f1efb715644d3368925903e9e94b9f243b21b97d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 10:45:15 +1000 Subject: [PATCH 076/109] Rename test --- tests/Feature/StructureTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index c269d39..07452c7 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -53,7 +53,7 @@ ]); }); -it('supports paginations with offset and 0 as first page', function () { +it('supports paginations with offset having 0 as first page', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->offset() ->firstPage(0) From b363fa10c7869a288ff1b547c63391809a6f91ec Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 10:45:46 +1000 Subject: [PATCH 077/109] Remove unneeded use statement --- src/Sources/LaravelClientResponse.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Sources/LaravelClientResponse.php b/src/Sources/LaravelClientResponse.php index bed169b..2490858 100644 --- a/src/Sources/LaravelClientResponse.php +++ b/src/Sources/LaravelClientResponse.php @@ -5,7 +5,6 @@ namespace Cerbero\LazyJsonPages\Sources; use Cerbero\LazyJsonPages\Exceptions\RequestNotSentException; -use Cerbero\LazyJsonPages\Services\Client; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Illuminate\Http\Client\Response; From ed81c947ce1290690c4df71300d272cb412b758b Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 12:18:13 +1000 Subject: [PATCH 078/109] Forget first response to save more memory --- src/Paginations/Pagination.php | 13 ++++--------- src/Sources/AnySource.php | 21 +++++++++++++++++++-- src/Sources/Endpoint.php | 14 ++------------ 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index c9cfd80..93709c5 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -4,11 +4,11 @@ namespace Cerbero\LazyJsonPages\Paginations; +use Cerbero\LazyJsonPages\Concerns\ParsesPages; use Cerbero\LazyJsonPages\Concerns\ResolvesPages; -use Cerbero\LazyJsonPages\Concerns\YieldsPaginatedItems; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Services\Book; -use Cerbero\LazyJsonPages\Sources\Source; +use Cerbero\LazyJsonPages\Sources\AnySource; use IteratorAggregate; use Traversable; @@ -19,7 +19,7 @@ */ abstract class Pagination implements IteratorAggregate { - use YieldsPaginatedItems; + use ParsesPages; use ResolvesPages; /** @@ -27,11 +27,6 @@ abstract class Pagination implements IteratorAggregate */ public readonly Book $book; - /** - * The number of items per page. - */ - protected readonly int $itemsPerPage; - /** * Yield the paginated items. * @@ -43,7 +38,7 @@ abstract public function getIterator(): Traversable; * Instantiate the class. */ final public function __construct( - protected readonly Source $source, + protected readonly AnySource $source, protected readonly Config $config, ) { $this->book = new Book(); diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index edd96c7..16b9269 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -28,7 +28,12 @@ class AnySource extends Source /** * The matching source. */ - protected ?Source $matchingSource; + protected readonly Source $matchingSource; + + /** + * The cached HTTP response. + */ + protected ?ResponseInterface $response; /** * Retrieve the HTTP request. @@ -67,6 +72,18 @@ protected function matchingSource(): Source */ public function response(): ResponseInterface { - return $this->matchingSource()->response(); + return $this->response ??= $this->matchingSource()->response(); + } + + /** + * Retrieve the HTTP response and forget it to save memory. + */ + public function pullResponse(): ResponseInterface + { + $response = $this->response(); + + $this->response = null; + + return $response; } } diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index 7eb81a9..3f55961 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -20,16 +20,6 @@ class Endpoint extends Source { use DetectsEndpoints; - /** - * The HTTP request. - */ - protected readonly RequestInterface $request; - - /** - * The HTTP response value object - */ - protected readonly ResponseInterface $response; - /** * Determine whether this class can handle the source. */ @@ -44,7 +34,7 @@ public function matches(): bool */ public function request(): RequestInterface { - return $this->request ??= new Request('GET', $this->source); + return new Request('GET', $this->source); } /** @@ -54,6 +44,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return $this->response ??= Client::instance()->send($this->request()); + return Client::instance()->send($this->request()); } } From 45a55b1b72b312285414e6c0273073f65c31dc9e Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 12:19:44 +1000 Subject: [PATCH 079/109] Improve methods reusability --- ...eldsPaginatedItems.php => ParsesPages.php} | 11 ++++-- src/Concerns/YieldsItemsByCursor.php | 36 ++++++++++++++++++ .../YieldsItemsByLength.php} | 37 ++++++++++--------- ...gination.php => CursorAwarePagination.php} | 20 +++++----- src/Paginations/CustomPagination.php | 2 +- src/Paginations/LastPageAwarePagination.php | 7 +++- src/Paginations/TotalItemsAwarePagination.php | 18 ++++----- src/Paginations/TotalPagesAwarePagination.php | 5 ++- 8 files changed, 89 insertions(+), 47 deletions(-) rename src/Concerns/{YieldsPaginatedItems.php => ParsesPages.php} (83%) create mode 100644 src/Concerns/YieldsItemsByCursor.php rename src/{Paginations/LengthAwarePagination.php => Concerns/YieldsItemsByLength.php} (51%) rename src/Paginations/{CursorPagination.php => CursorAwarePagination.php} (52%) diff --git a/src/Concerns/YieldsPaginatedItems.php b/src/Concerns/ParsesPages.php similarity index 83% rename from src/Concerns/YieldsPaginatedItems.php rename to src/Concerns/ParsesPages.php index 26759dd..6cdb426 100644 --- a/src/Concerns/YieldsPaginatedItems.php +++ b/src/Concerns/ParsesPages.php @@ -10,16 +10,21 @@ use Psr\Http\Message\ResponseInterface; /** - * The trait to yield paginated items. + * The trait to parse pages. */ -trait YieldsPaginatedItems +trait ParsesPages { + /** + * The number of items per page. + */ + protected readonly int $itemsPerPage; + /** * Yield paginated items and the given key from the provided response. * * @return Generator */ - protected function yieldItemsAndReturnKey(ResponseInterface $response, string $key): Generator + protected function yieldItemsAndGetKey(ResponseInterface $response, string $key): Generator { $itemsPerPage = 0; $pointers = [$this->config->itemsPointer]; diff --git a/src/Concerns/YieldsItemsByCursor.php b/src/Concerns/YieldsItemsByCursor.php new file mode 100644 index 0000000..bfb57fc --- /dev/null +++ b/src/Concerns/YieldsItemsByCursor.php @@ -0,0 +1,36 @@ +) $callback + * @return Generator + */ + protected function yieldItemsByCursor(Closure $callback): Generator + { + yield from $generator = $callback($this->source->pullResponse()); + + $request = clone $this->source->request(); + + while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { + $uri = $this->uriForPage($request->getUri(), (string) $cursor); + $response = Client::instance()->send($request->withUri($uri)); + + yield from $generator = $callback($response); + } + } +} diff --git a/src/Paginations/LengthAwarePagination.php b/src/Concerns/YieldsItemsByLength.php similarity index 51% rename from src/Paginations/LengthAwarePagination.php rename to src/Concerns/YieldsItemsByLength.php index 8709bb9..50a39b9 100644 --- a/src/Paginations/LengthAwarePagination.php +++ b/src/Concerns/YieldsItemsByLength.php @@ -2,51 +2,52 @@ declare(strict_types=1); -namespace Cerbero\LazyJsonPages\Paginations; +namespace Cerbero\LazyJsonPages\Concerns; -use Cerbero\LazyJsonPages\Concerns\SendsAsyncRequests; use Cerbero\LazyJsonPages\Exceptions\InvalidKeyException; use Closure; use Generator; use Illuminate\Support\LazyCollection; +use Psr\Http\Message\ResponseInterface; /** - * The abstract implementation of a pagination that is aware of its length. + * The trait to yield items from length-aware paginations. */ -abstract class LengthAwarePagination extends Pagination +trait YieldsItemsByLength { use SendsAsyncRequests; /** * Yield paginated items until the page resolved from the given key is reached. * - * @param (Closure(int): int)|null $callback + * @param (Closure(int): int) $callback * @return Generator */ - protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): Generator + protected function yieldItemsUntilKey(string $key, Closure $callback = null): Generator { - yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $key); + yield from $this->yieldItemsUntilPage(function(ResponseInterface $response) use ($key, $callback) { + yield from $generator = $this->yieldItemsAndGetKey($response, $key); - $page = $this->toPage($generator->getReturn()); + if (!is_int($page = $this->toPage($generator->getReturn()))) { + throw new InvalidKeyException($key); + } - if (!is_int($page)) { - throw new InvalidKeyException($key); - } - - $page = $callback ? $callback($page) : $page; - - yield from $this->yieldItemsUntilPage($page); + return $callback ? $callback($page) : $page; + }); } /** - * Yield paginated items until the given page is reached. + * Yield paginated items until the resolved page is reached. * + * @param (Closure(ResponseInterface): Generator) $callback * @return Generator */ - protected function yieldItemsUntilPage(int $page): Generator + protected function yieldItemsUntilPage(Closure $callback): Generator { + yield from $generator = $callback($this->source->pullResponse()); + $uri = $this->source->request()->getUri(); - $chunkedPages = $this->chunkPages($page); + $chunkedPages = $this->chunkPages($generator->getReturn()); foreach ($this->fetchPagesAsynchronously($chunkedPages, $uri) as $page) { yield from $this->yieldItemsFrom($page); diff --git a/src/Paginations/CursorPagination.php b/src/Paginations/CursorAwarePagination.php similarity index 52% rename from src/Paginations/CursorPagination.php rename to src/Paginations/CursorAwarePagination.php index ffa9812..19b9de6 100644 --- a/src/Paginations/CursorPagination.php +++ b/src/Paginations/CursorAwarePagination.php @@ -4,14 +4,17 @@ namespace Cerbero\LazyJsonPages\Paginations; -use Cerbero\LazyJsonPages\Services\Client; +use Cerbero\LazyJsonPages\Concerns\YieldsItemsByCursor; +use Psr\Http\Message\ResponseInterface; use Traversable; /** * The pagination aware of the cursor of the next page. */ -class CursorPagination extends Pagination +class CursorAwarePagination extends Pagination { + use YieldsItemsByCursor; + /** * Determine whether the configuration matches this pagination. */ @@ -30,15 +33,10 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->cursorKey); - - $request = clone $this->source->request(); - - while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { - $uri = $this->uriForPage($request->getUri(), (string) $cursor); - $response = Client::instance()->send($request->withUri($uri)); + yield from $this->yieldItemsByCursor(function(ResponseInterface $response) { + yield from $generator = $this->yieldItemsAndGetKey($response, $this->config->cursorKey); - yield from $generator = $this->yieldItemsAndReturnKey($response, $this->config->cursorKey); - } + return $generator->getReturn(); + }); } } diff --git a/src/Paginations/CustomPagination.php b/src/Paginations/CustomPagination.php index 18339f2..c92eb74 100644 --- a/src/Paginations/CustomPagination.php +++ b/src/Paginations/CustomPagination.php @@ -10,7 +10,7 @@ /** * The user-defined pagination. */ -class CustomPagination extends LengthAwarePagination +class CustomPagination extends Pagination { /** * Determine whether the configuration matches this pagination. diff --git a/src/Paginations/LastPageAwarePagination.php b/src/Paginations/LastPageAwarePagination.php index a764f23..bd259b0 100644 --- a/src/Paginations/LastPageAwarePagination.php +++ b/src/Paginations/LastPageAwarePagination.php @@ -4,13 +4,16 @@ namespace Cerbero\LazyJsonPages\Paginations; +use Cerbero\LazyJsonPages\Concerns\YieldsItemsByLength; use Traversable; /** * The pagination aware of the number of the last page. */ -class LastPageAwarePagination extends LengthAwarePagination +class LastPageAwarePagination extends Pagination { + use YieldsItemsByLength; + /** * Determine whether the configuration matches this pagination. */ @@ -26,7 +29,7 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function (int $page) { + yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function(int $page) { return $this->config->firstPage === 0 ? $page + 1 : $page; }); } diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 21087c8..049b269 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -4,14 +4,16 @@ namespace Cerbero\LazyJsonPages\Paginations; -use Cerbero\LazyJsonPages\Exceptions\InvalidKeyException; +use Cerbero\LazyJsonPages\Concerns\YieldsItemsByLength; use Traversable; /** * The pagination aware of the total number of items. */ -class TotalItemsAwarePagination extends LengthAwarePagination +class TotalItemsAwarePagination extends Pagination { + use YieldsItemsByLength; + /** * Determine whether the configuration matches this pagination. */ @@ -29,14 +31,8 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $generator = $this->yieldItemsAndReturnKey($this->source->response(), $this->config->totalItemsKey); - - if (!is_numeric($totalItems = $generator->getReturn())) { - throw new InvalidKeyException($this->config->totalItemsKey); - } - - $totalPages = $this->itemsPerPage > 0 ? (int) ceil(intval($totalItems) / $this->itemsPerPage) : 0; - - yield from $this->yieldItemsUntilPage($totalPages); + yield from $this->yieldItemsUntilKey($this->config->totalItemsKey, function(int $totalItems) { + return $this->itemsPerPage > 0 ? (int) ceil($totalItems / $this->itemsPerPage) : 0; + }); } } diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index c2aeabe..d216ac2 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -4,13 +4,16 @@ namespace Cerbero\LazyJsonPages\Paginations; +use Cerbero\LazyJsonPages\Concerns\YieldsItemsByLength; use Traversable; /** * The pagination aware of the total number of pages. */ -class TotalPagesAwarePagination extends LengthAwarePagination +class TotalPagesAwarePagination extends Pagination { + use YieldsItemsByLength; + /** * Determine whether the configuration matches this pagination. */ From 1e5475795740719b062bf5bd08ea740a85e32587 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 12:20:57 +1000 Subject: [PATCH 080/109] Implement link header aware pagination --- src/Dtos/Config.php | 1 + src/LazyJsonPages.php | 12 +- src/Paginations/AnyPagination.php | 4 +- src/Paginations/LinkHeaderAwarePagination.php | 104 ++++++++++++++++++ tests/Feature/Datasets.php | 1 + tests/Feature/PaginationTest.php | 26 +++++ tests/Pest.php | 15 +-- 7 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 src/Paginations/LinkHeaderAwarePagination.php diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 3106e27..0c13352 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -27,6 +27,7 @@ public function __construct( public readonly ?string $cursorKey = null, public readonly ?string $lastPageKey = null, public readonly ?string $offsetKey = null, + public readonly bool $hasLinkHeader = false, public readonly ?string $pagination = null, public readonly int $async = 3, public readonly int $attempts = 3, diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 99fb3e7..1d95a3d 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -135,6 +135,16 @@ public function offset(string $key = 'offset'): self return $this; } + /** + * Set the Link header pagination. + */ + public function linkHeader(): self + { + $this->config['hasLinkHeader'] = true; + + return $this; + } + /** * Set the custom pagination. */ @@ -216,7 +226,7 @@ public function collect(string $dot = '*'): LazyCollection $config = new Config(...$this->config, itemsPointer: DotsConverter::toPointer($dot)); - return new LazyCollection(function () use ($config) { + return new LazyCollection(function() use ($config) { try { yield from new AnyPagination($this->source, $config); } finally { diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index b2987e2..1b51ac0 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -18,10 +18,10 @@ class AnyPagination extends Pagination * @var class-string[] */ protected array $supportedPaginations = [ - CursorPagination::class, + CursorAwarePagination::class, CustomPagination::class, LastPageAwarePagination::class, - // LinkHeaderPagination::class, + LinkHeaderAwarePagination::class, TotalItemsAwarePagination::class, TotalPagesAwarePagination::class, ]; diff --git a/src/Paginations/LinkHeaderAwarePagination.php b/src/Paginations/LinkHeaderAwarePagination.php new file mode 100644 index 0000000..a380b50 --- /dev/null +++ b/src/Paginations/LinkHeaderAwarePagination.php @@ -0,0 +1,104 @@ +[^\s>]+)\s*>.*?"\s*(?[^\s"]+)\s*"~'; + + /** + * Determine whether the configuration matches this pagination. + */ + public function matches(): bool + { + return $this->config->hasLinkHeader + && $this->config->totalItemsKey === null + && $this->config->totalPagesKey === null + && $this->config->lastPageKey === null; + } + + /** + * Yield the paginated items. + * + * @return Traversable + * @throws InvalidLinkHeaderException + */ + public function getIterator(): Traversable + { + $links = $this->parseLinkHeader($this->source->response()->getHeaderLine('link')); + + yield from match (true) { + isset($links['last']) => $this->yieldItemsByLastPage($links['last']), + isset($links['next']) => $this->yieldItemsByNextLink(), + default => $this->yieldItemsFrom($this->source->pullResponse()), + }; + } + + /** + * Retrieve the parsed Link header. + * + * @template TParsed of array{last?: int, next?: string|int} + * @template TRelation of string|null + * + * @param TRelation $relation + * @return (TRelation is null ? TParsed : string|int|null) + */ + protected function parseLinkHeader(string $linkHeader, ?string $relation = null): array|string|int|null + { + $links = []; + + preg_match_all(static::FORMAT, $linkHeader, $matches, PREG_SET_ORDER); + + foreach ($matches as $match) { + $links[$match['rel']] = $this->toPage($match['uri'], $match['rel'] != 'next'); + } + + return $relation ? ($links[$relation] ?? null) : $links; + } + + /** + * Yield the paginated items by the given last page. + * + * @return Generator + */ + protected function yieldItemsByLastPage(int $lastPage): Generator + { + yield from $this->yieldItemsUntilPage(function(ResponseInterface $response) use ($lastPage) { + yield from $this->yieldItemsFrom($response); + + return $this->config->firstPage === 0 ? $lastPage + 1 : $lastPage; + }); + } + + /** + * Yield the paginated items by the given next link. + * + * @return Generator + */ + protected function yieldItemsByNextLink(): Generator + { + yield from $this->yieldItemsByCursor(function(ResponseInterface $response) { + yield from $this->yieldItemsFrom($response); + + return $this->parseLinkHeader($response->getHeaderLine('link'), 'next'); + }); + } +} diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index 369b435..45ad9b6 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -31,6 +31,7 @@ yield 'total pages aware' => fn(LazyJsonPages $instance) => $instance->totalPages('meta.total_pages'); yield 'total items aware' => fn(LazyJsonPages $instance) => $instance->totalItems('meta.total_items'); yield 'last page aware' => fn(LazyJsonPages $instance) => $instance->lastPage('meta.last_page'); + yield 'link header' => fn(LazyJsonPages $instance) => $instance->linkHeader(); yield 'custom pagination' => fn(LazyJsonPages $instance) => $instance->pagination(TotalPagesAwarePagination::class)->totalPages('meta.total_pages'); yield 'total pages aware by header' => fn(LazyJsonPages $instance) => $instance->totalPages('X-Total-Pages'); yield 'total items aware by header' => fn(LazyJsonPages $instance) => $instance->totalItems('X-Total-Items'); diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index aa66ff8..0d0a67b 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -11,6 +11,11 @@ 'https://example.com/api/v1/users' => 'pagination/page1.json', 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', + ], headers: [ + 'X-Total-Pages' => 3, + 'X-Total-Items' => 14, + 'X-Last-Page' => 3, + 'Link' => ';rel="last"', ]); })->with('length-aware'); @@ -23,6 +28,11 @@ 'https://example.com/api/v1/users' => 'paginationFirstPage0/page0.json', 'https://example.com/api/v1/users?page=1' => 'paginationFirstPage0/page1.json', 'https://example.com/api/v1/users?page=2' => 'paginationFirstPage0/page2.json', + ], headers: [ + 'X-Total-Pages' => 3, + 'X-Total-Items' => 14, + 'X-Last-Page' => 2, + 'Link' => ';rel="last"', ]); })->with('length-aware'); @@ -38,6 +48,22 @@ ]); }); +it('supports cursor-aware paginations with link header', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->linkHeader() + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=cursor1' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=cursor2' => 'pagination/page3.json', + ], headers: (function() { + yield ['Link' => ';rel="next"']; + yield ['Link' => ';rel="next"']; + yield ['Link' => '']; + })()); +}); + it('fails if an invalid custom pagination is provided', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->pagination('Invalid') diff --git a/tests/Pest.php b/tests/Pest.php index 3f985fd..f91e43a 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -30,16 +30,17 @@ | */ -expect()->extend('toLoadItemsViaRequests', function (array $requests) { +expect()->extend('toLoadItemsViaRequests', function (array $requests, Generator|array $headers = []) { $responses = $transactions = $expectedUris = []; - $headers = [ - 'X-Total-Pages' => 3, - 'X-Total-Items' => 14, - ]; + $responseHeaders = $headers; foreach ($requests as $uri => $fixture) { - $headers['X-Last-Page'] ??= str_contains($fixture, 'page0') ? 2 : 3; - $responses[] = new Response(body: file_get_contents(fixture($fixture)), headers: $headers); + if ($headers instanceof Generator) { + $responseHeaders = $headers->current(); + $headers->valid() && $headers->next(); + } + + $responses[] = new Response(body: file_get_contents(fixture($fixture)), headers: $responseHeaders); $expectedUris[] = $uri; } From dfdf383e4943e4671e69c345cb53ac2702786682 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 20:30:57 +1000 Subject: [PATCH 081/109] Configure testbench --- .gitignore | 1 + testbench.yaml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 testbench.yaml diff --git a/.gitignore b/.gitignore index bd2b70e..c9c70a3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ composer.lock vendor phpcs.xml phpunit.xml +.phpunit.cache .phpunit.result.cache .DS_Store diff --git a/testbench.yaml b/testbench.yaml new file mode 100644 index 0000000..9ac32c6 --- /dev/null +++ b/testbench.yaml @@ -0,0 +1,2 @@ +providers: + - Cerbero\LazyJsonPages\Providers\LazyJsonPagesServiceProvider From 053c419a6af1b7aac1e794b70b742c1cb8fec32c Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 20:32:51 +1000 Subject: [PATCH 082/109] Integrate Laravel HTTP client events --- composer.json | 6 +++ .../LazyJsonPagesServiceProvider.php | 52 +++++++++++++++++++ src/Services/Client.php | 34 ++++++++++-- tests/Integration/LaravelTest.php | 37 +++++++++++++ tests/Pest.php | 27 +++++++++- 5 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 src/Providers/LazyJsonPagesServiceProvider.php create mode 100644 tests/Integration/LaravelTest.php diff --git a/composer.json b/composer.json index f927fbc..f72340c 100644 --- a/composer.json +++ b/composer.json @@ -26,6 +26,7 @@ }, "require-dev": { "illuminate/http": ">=6.20", + "illuminate/support": ">=6.20", "mockery/mockery": "^1.3.4", "orchestra/testbench": ">=7.0", "pestphp/pest": "^2.0", @@ -55,6 +56,11 @@ "extra": { "branch-alias": { "dev-master": "1.0-dev" + }, + "laravel": { + "providers": [ + "Cerbero\\LazyJsonPages\\Providers\\LazyJsonPagesServiceProvider" + ] } }, "config": { diff --git a/src/Providers/LazyJsonPagesServiceProvider.php b/src/Providers/LazyJsonPagesServiceProvider.php new file mode 100644 index 0000000..fbf8d7d --- /dev/null +++ b/src/Providers/LazyJsonPagesServiceProvider.php @@ -0,0 +1,52 @@ +sending(...), $this->sent(...))); + } + + /** + * Handle HTTP requests before they are sent. + */ + private function sending(RequestInterface $request): void + { + event(new RequestSending(new Request($request))); + } + + /** + * Handle HTTP requests after they are sent. + */ + private function sent(RequestInterface $request, array $options, PromiseInterface $promise): void + { + $clientRequest = new Request($request); + + $promise->then( + fn(ResponseInterface $response) => event(new ResponseReceived($clientRequest, new Response($response))), + fn() => event(new ConnectionFailed($clientRequest)), + ); + } +} diff --git a/src/Services/Client.php b/src/Services/Client.php index 793aed0..ec2e7f2 100644 --- a/src/Services/Client.php +++ b/src/Services/Client.php @@ -5,6 +5,7 @@ namespace Cerbero\LazyJsonPages\Services; use GuzzleHttp\Client as Guzzle; +use GuzzleHttp\HandlerStack; use GuzzleHttp\RequestOptions; /** @@ -27,9 +28,18 @@ final class Client /** * The custom options. + * + * @var array */ private static array $options = []; + /** + * The client middleware. + * + * @var array + */ + private static array $middleware = []; + /** * The Guzzle client instance. */ @@ -40,9 +50,18 @@ final class Client */ public static function instance(): Guzzle { - return self::$guzzle ??= new Guzzle( - array_replace_recursive(self::$defaultOptions, self::$options), - ); + if (self::$guzzle) { + return self::$guzzle; + } + + $options = array_replace_recursive(self::$defaultOptions, self::$options); + $options['handler'] ??= HandlerStack::create(); + + foreach (self::$middleware as $name => $middleware) { + $options['handler']->push($middleware, $name); + } + + return self::$guzzle = new Guzzle($options); } /** @@ -53,6 +72,14 @@ public static function configure(array $options): void self::$options = array_replace_recursive(self::$options, $options); } + /** + * Set the Guzzle client middleware. + */ + public static function middleware(string $name, callable $middleware): void + { + self::$middleware[$name] = $middleware; + } + /** * Clean up the static values. */ @@ -60,6 +87,7 @@ public static function reset(): void { self::$guzzle = null; self::$options = []; + self::$middleware = []; } /** diff --git a/tests/Integration/LaravelTest.php b/tests/Integration/LaravelTest.php new file mode 100644 index 0000000..7bef011 --- /dev/null +++ b/tests/Integration/LaravelTest.php @@ -0,0 +1,37 @@ +totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', + ]); + + Event::assertDispatched(RequestSending::class, 3); + Event::assertDispatched(ResponseReceived::class, 3); +}); + +it('fires Laravel HTTP client events on failure', function() { + Event::fake(); + + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toFailRequest('https://example.com/api/v1/users'); + + Event::assertDispatched(RequestSending::class, 1); + Event::assertDispatched(ConnectionFailed::class, 1); +}); diff --git a/tests/Pest.php b/tests/Pest.php index f91e43a..ec36cc9 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,10 +1,14 @@ in('Feature'); +uses(OrchestraTestCase::class, WithWorkbench::class)->in('Integration/LaravelTest.php'); /* |-------------------------------------------------------------------------- @@ -57,6 +61,27 @@ expect($actualUris)->toBe($expectedUris); }); +expect()->extend('toFailRequest', function (string $uri) { + $transactions = []; + + $responses = [$exception = new RequestException('connection failed', new Request('GET', $uri))]; + + $stack = HandlerStack::create(new MockHandler($responses)); + + $stack->push(Middleware::history($transactions)); + + Client::configure(['handler' => $stack]); + + try { + iterator_to_array($this->value); + } catch (Throwable $e) { + expect($e)->toBe($exception); + } + + expect($transactions)->toHaveCount(1); + expect((string) $transactions[0]['request']->getUri())->toBe($uri); +}); + /* |-------------------------------------------------------------------------- | Functions From 84650a2baf98ddd7df72ccb2fd08b51df118e6f1 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 11 Feb 2024 20:53:26 +1000 Subject: [PATCH 083/109] Update readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 9b66239..3d2e71e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ composer require cerbero/lazy-json-pages * [๐Ÿ‘ฝ Custom paginations](#-custom-paginations) * [๐Ÿš€ Requests optimization](#-requests-optimization) * [๐Ÿ’ข Errors handling](#-errors-handling) +* [๐Ÿค Laravel integration](#-laravel-integration) ### ๐Ÿ‘ฃ Basics @@ -288,6 +289,17 @@ If you find yourself implementing the same custom pagination in different projec > [!WARNING] > The documentation of this feature is a work in progress. + +### ๐Ÿค Laravel integration + +If used in a [Laravel](https://laravel.com) project, Lazy JSON Pages automatically fires events when: +- an HTTP request is about to be sent, by firing `Illuminate\Http\Client\Events\RequestSending` +- an HTTP response is received, by firing `Illuminate\Http\Client\Events\ResponseReceived` +- a connection failed, by firing `Illuminate\Http\Client\Events\ConnectionFailed` + +This is especially handy for debugging tools like [Laravel Telescope](https://laravel.com/docs/telescope) or [Spatie Ray](https://spatie.be/docs/ray/installation-in-your-project/laravel) or for triggering the related event listeners. + + ## ๐Ÿ“† Change log Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. From dbb4eb95de6dfe3063e2a2da7c7f9fadff9aec24 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 12 Feb 2024 15:42:35 +1000 Subject: [PATCH 084/109] Share a client per instance instead of a singleton --- README.md | 28 +++- src/Concerns/SendsAsyncRequests.php | 3 +- src/Concerns/YieldsItemsByCursor.php | 3 +- src/LazyJsonPages.php | 69 ++++++---- src/Paginations/AnyPagination.php | 2 +- src/Paginations/CustomPagination.php | 2 +- src/Paginations/Pagination.php | 2 + .../LazyJsonPagesServiceProvider.php | 4 +- src/Services/Client.php | 100 -------------- src/Services/ClientFactory.php | 122 ++++++++++++++++++ src/Sources/AnySource.php | 2 +- src/Sources/Endpoint.php | 3 +- src/Sources/LaravelClientRequest.php | 3 +- src/Sources/Psr7Request.php | 3 +- src/Sources/Source.php | 2 + src/Sources/SymfonyRequest.php | 3 +- tests/Feature/Datasets.php | 3 +- tests/Feature/OptimizationTest.php | 24 ++++ tests/Pest.php | 36 ++---- 19 files changed, 242 insertions(+), 172 deletions(-) delete mode 100644 src/Services/Client.php create mode 100644 src/Services/ClientFactory.php create mode 100644 tests/Feature/OptimizationTest.php diff --git a/README.md b/README.md index 3d2e71e..0e957f1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ $lazyCollection = LazyJsonPages::from($source) Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. > [!TIP] -> Need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. +> Do you need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. ## ๐Ÿ“ฆ Install @@ -49,7 +49,7 @@ composer require cerbero/lazy-json-pages ### ๐Ÿ‘ฃ Basics -Depending on our coding style, we can call Lazy JSON Pages in 3 different ways: +Depending on our coding style, we can initialize Lazy JSON Pages in 3 different ways: ```php use Cerbero\LazyJsonPages\LazyJsonPages; @@ -123,7 +123,9 @@ class CustomSource extends Source } ``` -The parent class `Source` gives us access to the property `$source`, which is the custom source for our use case. +The parent class `Source` gives us access to 2 properties: +- `$source`: the custom source for our use case +- `$client`: the Guzzle HTTP client The methods to implement respectively turn our custom source into a PSR-7 request and a PSR-7 response. Please refer to the [already existing sources](https://github.com/cerbero90/json-parser/tree/master/src/Sources) to see some implementations. @@ -263,8 +265,9 @@ class CustomPagination extends Pagination } ``` -The parent class `Pagination` gives us access to 2 properties: +The parent class `Pagination` gives us access to 3 properties: - `$source`: the [source](#-sources) pointing to the paginated JSON API +- `$client`: the Guzzle HTTP client - `$config`: the configuration that we generated by chaining methods like `totalPages()` The method `getIterator()` defines the logic to extract paginated items in a memory-efficient way. Please refer to the [already existing paginations](https://github.com/cerbero90/json-parser/tree/master/src/Paginations) to see some implementations. @@ -280,8 +283,21 @@ If you find yourself implementing the same custom pagination in different projec ### ๐Ÿš€ Requests optimization -> [!WARNING] -> The documentation of this feature is a work in progress. +Paginated APIs differ from each other, so Lazy JSON Pages lets us tweak our HTTP requests specifically for our use case. + +Internally, Lazy JSON Pages uses [Guzzle](https://docs.guzzlephp.org) as its HTTP client. We can customize the client behavior by adding as many [middleware](https://docs.guzzlephp.org/en/stable/handlers-and-middleware.html#middleware) as we need: + +```php +LazyJsonPages::from($source) + ->middleware('log_requests', $logRequests) + ->middleware('cache_responses', $cacheResponses); +``` + +If we a middleware to be added every time we invoke Lazy JSON Pages, we can add a global middleware: + +```php +LazyJsonPages::globalMiddleware('fire_events', $fireEvents); +``` ### ๐Ÿ’ข Errors handling diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 325d96b..cdd11ae 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJsonPages\Concerns; -use Cerbero\LazyJsonPages\Services\Client; use Generator; use GuzzleHttp\Pool; use Illuminate\Support\LazyCollection; @@ -44,7 +43,7 @@ protected function fetchPagesAsynchronously(LazyCollection $chunkedPages, UriInt */ protected function pool(UriInterface $uri, array $pages): Pool { - return new Pool(Client::instance(), $this->yieldRequests($uri, $pages), [ + return new Pool($this->client, $this->yieldRequests($uri, $pages), [ 'concurrency' => $this->config->async, 'fulfilled' => fn(ResponseInterface $response, int $page) => $this->book->addPage($page, $response), 'rejected' => fn(Throwable $e, int $page) => $this->book->addFailedPage($page) && throw $e, diff --git a/src/Concerns/YieldsItemsByCursor.php b/src/Concerns/YieldsItemsByCursor.php index bfb57fc..ec8c1c2 100644 --- a/src/Concerns/YieldsItemsByCursor.php +++ b/src/Concerns/YieldsItemsByCursor.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJsonPages\Concerns; -use Cerbero\LazyJsonPages\Services\Client; use Closure; use Generator; use Psr\Http\Message\ResponseInterface; @@ -28,7 +27,7 @@ protected function yieldItemsByCursor(Closure $callback): Generator while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { $uri = $this->uriForPage($request->getUri(), (string) $cursor); - $response = Client::instance()->send($request->withUri($uri)); + $response = $this->client->send($request->withUri($uri)); yield from $generator = $callback($response); } diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 1d95a3d..3677ff9 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -7,7 +7,7 @@ use Cerbero\LazyJson\Pointers\DotsConverter; use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Paginations\AnyPagination; -use Cerbero\LazyJsonPages\Services\Client; +use Cerbero\LazyJsonPages\Services\ClientFactory; use Cerbero\LazyJsonPages\Sources\AnySource; use Closure; use GuzzleHttp\RequestOptions; @@ -19,9 +19,27 @@ final class LazyJsonPages { /** - * The source of the paginated API. + * The HTTP client factory. */ - private readonly AnySource $source; + private readonly ClientFactory $factory; + + /** + * The Guzzle HTTP request options. + * + * @var array + */ + private array $options = [ + RequestOptions::CONNECT_TIMEOUT => 5, + RequestOptions::READ_TIMEOUT => 5, + RequestOptions::TIMEOUT => 5, + ]; + + /** + * The Guzzle client middleware. + * + * @var array + */ + private array $middleware = []; /** * The raw configuration of the API pagination. @@ -31,13 +49,12 @@ final class LazyJsonPages private array $config = []; /** - * The Guzzle HTTP request options. + * Add a global middleware. */ - private array $requestOptions = [ - RequestOptions::CONNECT_TIMEOUT => 5, - RequestOptions::READ_TIMEOUT => 5, - RequestOptions::TIMEOUT => 5, - ]; + public static function globalMiddleware(string $name, callable $middleware): void + { + ClientFactory::globalMiddleware($name, $middleware); + } /** * Instantiate the class statically. @@ -50,9 +67,9 @@ public static function from(mixed $source): self /** * Instantiate the class. */ - public function __construct(mixed $source) + public function __construct(private readonly mixed $source) { - $this->source = new AnySource($source); + $this->factory = new ClientFactory(); } /** @@ -178,7 +195,7 @@ public function async(int $requests): self */ public function connectionTimeout(float|int $seconds): self { - $this->requestOptions[RequestOptions::CONNECT_TIMEOUT] = max(0, $seconds); + $this->options[RequestOptions::CONNECT_TIMEOUT] = max(0, $seconds); return $this; } @@ -188,8 +205,8 @@ public function connectionTimeout(float|int $seconds): self */ public function requestTimeout(float|int $seconds): self { - $this->requestOptions[RequestOptions::TIMEOUT] = max(0, $seconds); - $this->requestOptions[RequestOptions::READ_TIMEOUT] = max(0, $seconds); + $this->options[RequestOptions::TIMEOUT] = max(0, $seconds); + $this->options[RequestOptions::READ_TIMEOUT] = max(0, $seconds); return $this; } @@ -214,6 +231,16 @@ public function backoff(Closure $callback): self return $this; } + /** + * Add an HTTP client middleware. + */ + public function middleware(string $name, callable $middleware): self + { + $this->middleware[$name] = $middleware; + + return $this; + } + /** * Retrieve a lazy collection yielding the paginated items. * @@ -222,16 +249,12 @@ public function backoff(Closure $callback): self */ public function collect(string $dot = '*'): LazyCollection { - Client::configure($this->requestOptions); - - $config = new Config(...$this->config, itemsPointer: DotsConverter::toPointer($dot)); + return new LazyCollection(function() use ($dot) { + $config = new Config(...$this->config, itemsPointer: DotsConverter::toPointer($dot)); + $client = $this->factory->options($this->options)->middleware($this->middleware)->make(); + $source = new AnySource($this->source, $client); - return new LazyCollection(function() use ($config) { - try { - yield from new AnyPagination($this->source, $config); - } finally { - Client::reset(); - } + yield from new AnyPagination($source, $client, $config); }); } } diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 1b51ac0..0d31c9b 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -47,7 +47,7 @@ public function getIterator(): Traversable protected function matchingPagination(): Pagination { foreach ($this->supportedPaginations as $class) { - $pagination = new $class($this->source, $this->config); + $pagination = new $class($this->source, $this->client, $this->config); if ($pagination->matches()) { return $pagination; diff --git a/src/Paginations/CustomPagination.php b/src/Paginations/CustomPagination.php index c92eb74..82b6983 100644 --- a/src/Paginations/CustomPagination.php +++ b/src/Paginations/CustomPagination.php @@ -31,6 +31,6 @@ public function getIterator(): Traversable throw new InvalidPaginationException($this->config->pagination); } - yield from new $this->config->pagination($this->source, $this->config); + yield from new ($this->config->pagination)($this->source, $this->client, $this->config); } } diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 93709c5..1d07c21 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -9,6 +9,7 @@ use Cerbero\LazyJsonPages\Dtos\Config; use Cerbero\LazyJsonPages\Services\Book; use Cerbero\LazyJsonPages\Sources\AnySource; +use GuzzleHttp\Client; use IteratorAggregate; use Traversable; @@ -39,6 +40,7 @@ abstract public function getIterator(): Traversable; */ final public function __construct( protected readonly AnySource $source, + protected readonly Client $client, protected readonly Config $config, ) { $this->book = new Book(); diff --git a/src/Providers/LazyJsonPagesServiceProvider.php b/src/Providers/LazyJsonPagesServiceProvider.php index fbf8d7d..9c7fff8 100644 --- a/src/Providers/LazyJsonPagesServiceProvider.php +++ b/src/Providers/LazyJsonPagesServiceProvider.php @@ -4,7 +4,7 @@ namespace Cerbero\LazyJsonPages\Providers; -use Cerbero\LazyJsonPages\Services\Client; +use Cerbero\LazyJsonPages\LazyJsonPages; use GuzzleHttp\Middleware; use GuzzleHttp\Promise\PromiseInterface; use Illuminate\Http\Client\Events\ConnectionFailed; @@ -26,7 +26,7 @@ final class LazyJsonPagesServiceProvider extends ServiceProvider */ public function boot(): void { - Client::middleware('laravel_events', Middleware::tap($this->sending(...), $this->sent(...))); + LazyJsonPages::globalMiddleware('laravel_events', Middleware::tap($this->sending(...), $this->sent(...))); } /** diff --git a/src/Services/Client.php b/src/Services/Client.php deleted file mode 100644 index ec2e7f2..0000000 --- a/src/Services/Client.php +++ /dev/null @@ -1,100 +0,0 @@ - - */ - private static array $defaultOptions = [ - RequestOptions::STREAM => true, - RequestOptions::HEADERS => [ - 'Accept' => 'application/json', - 'Content-Type' => 'application/json', - ], - ]; - - /** - * The custom options. - * - * @var array - */ - private static array $options = []; - - /** - * The client middleware. - * - * @var array - */ - private static array $middleware = []; - - /** - * The Guzzle client instance. - */ - private static ?Guzzle $guzzle = null; - - /** - * Retrieve the Guzzle client instance. - */ - public static function instance(): Guzzle - { - if (self::$guzzle) { - return self::$guzzle; - } - - $options = array_replace_recursive(self::$defaultOptions, self::$options); - $options['handler'] ??= HandlerStack::create(); - - foreach (self::$middleware as $name => $middleware) { - $options['handler']->push($middleware, $name); - } - - return self::$guzzle = new Guzzle($options); - } - - /** - * Set the Guzzle client options. - */ - public static function configure(array $options): void - { - self::$options = array_replace_recursive(self::$options, $options); - } - - /** - * Set the Guzzle client middleware. - */ - public static function middleware(string $name, callable $middleware): void - { - self::$middleware[$name] = $middleware; - } - - /** - * Clean up the static values. - */ - public static function reset(): void - { - self::$guzzle = null; - self::$options = []; - self::$middleware = []; - } - - /** - * Instantiate the class. - */ - private function __construct() - { - // disable the constructor - } -} diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php new file mode 100644 index 0000000..05b4142 --- /dev/null +++ b/src/Services/ClientFactory.php @@ -0,0 +1,122 @@ + + */ + private static array $defaultOptions = [ + RequestOptions::STREAM => true, + RequestOptions::HEADERS => [ + 'Accept' => 'application/json', + 'Content-Type' => 'application/json', + ], + ]; + + /** + * The global middleware. + * + * @var array + */ + private static array $globalMiddleware = []; + + /** + * The custom options. + * + * @var array + */ + private array $options = []; + + /** + * The local middleware. + * + * @var array + */ + private array $middleware = []; + + /** + * Add a global middleware. + */ + public static function globalMiddleware(string $name, callable $middleware): void + { + self::$globalMiddleware[$name] = $middleware; + } + + /** + * Fake HTTP requests for testing purposes. + * + * @param \Psr\Http\Message\ResponseInterface[]|GuzzleHttp\Exception\RequestException[] $responses + * @return array> + */ + public static function fake(array $responses, Closure $callback): array + { + $transactions = []; + + $handler = HandlerStack::create(new MockHandler($responses)); + + $handler->push(Middleware::history($transactions)); + + self::$defaultOptions['handler'] = $handler; + + $callback(); + + unset(self::$defaultOptions['handler']); + + return $transactions; + } + + /** + * Set the Guzzle client options. + * + * @param array $options + */ + public function options(array $options): self + { + $this->options = $options; + + return $this; + } + + /** + * Set the Guzzle client middleware. + * + * @param array $middleware + */ + public function middleware(array $middleware): self + { + $this->middleware = $middleware; + + return $this; + } + + /** + * Retrieve a configured Guzzle client instance. + */ + public function make(): Client + { + $options = array_replace_recursive(self::$defaultOptions, $this->options); + $options['handler'] ??= HandlerStack::create(); + + foreach ([...self::$globalMiddleware, ...$this->middleware] as $name => $middleware) { + $options['handler']->push($middleware, $name); + } + + return new Client($options); + } +} diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index 16b9269..c37769b 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -55,7 +55,7 @@ protected function matchingSource(): Source } foreach ($this->supportedSources as $class) { - $source = new $class($this->source); + $source = new $class($this->source, $this->client); if ($source->matches()) { return $this->matchingSource = $source; diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index 3f55961..74ecb31 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -5,7 +5,6 @@ namespace Cerbero\LazyJsonPages\Sources; use Cerbero\JsonParser\Concerns\DetectsEndpoints; -use Cerbero\LazyJsonPages\Services\Client; use GuzzleHttp\Psr7\Request; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -44,6 +43,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return Client::instance()->send($this->request()); + return $this->client->send($this->request()); } } diff --git a/src/Sources/LaravelClientRequest.php b/src/Sources/LaravelClientRequest.php index 072674c..ed1eb85 100644 --- a/src/Sources/LaravelClientRequest.php +++ b/src/Sources/LaravelClientRequest.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJsonPages\Sources; -use Cerbero\LazyJsonPages\Services\Client; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Illuminate\Http\Client\Request; @@ -39,6 +38,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return Client::instance()->send($this->request()); + return $this->client->send($this->request()); } } diff --git a/src/Sources/Psr7Request.php b/src/Sources/Psr7Request.php index 35821d9..955b90f 100644 --- a/src/Sources/Psr7Request.php +++ b/src/Sources/Psr7Request.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJsonPages\Sources; -use Cerbero\LazyJsonPages\Services\Client; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -38,6 +37,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return Client::instance()->send($this->source); + return $this->client->send($this->source); } } diff --git a/src/Sources/Source.php b/src/Sources/Source.php index b337f48..d261817 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -4,6 +4,7 @@ namespace Cerbero\LazyJsonPages\Sources; +use GuzzleHttp\Client; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -29,6 +30,7 @@ abstract public function response(): ResponseInterface; */ final public function __construct( protected readonly mixed $source, + protected readonly Client $client, ) {} /** diff --git a/src/Sources/SymfonyRequest.php b/src/Sources/SymfonyRequest.php index 69166c5..fba5ebc 100644 --- a/src/Sources/SymfonyRequest.php +++ b/src/Sources/SymfonyRequest.php @@ -4,7 +4,6 @@ namespace Cerbero\LazyJsonPages\Sources; -use Cerbero\LazyJsonPages\Services\Client; use GuzzleHttp\Psr7\Request as Psr7Request; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -45,6 +44,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return Client::instance()->send($this->request()); + return $this->client->send($this->request()); } } diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index 45ad9b6..ed4e439 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -2,6 +2,7 @@ use Cerbero\LazyJsonPages\LazyJsonPages; use Cerbero\LazyJsonPages\Paginations\TotalPagesAwarePagination; +use Cerbero\LazyJsonPages\Services\ClientFactory; use Cerbero\LazyJsonPages\Sources\CustomSourceSample; use GuzzleHttp\Psr7\Request as Psr7Request; use GuzzleHttp\Psr7\Response as Psr7Response; @@ -18,7 +19,7 @@ $laravelClientResponse = new LaravelClientResponse($psr7Response); $laravelClientResponse->transferStats = new TransferStats($psr7Request, $psr7Response); - yield 'user-defined source' => [new CustomSourceSample(null), false]; + yield 'user-defined source' => [new CustomSourceSample(null, (new ClientFactory())->make()), false]; yield 'endpoint' => [$uri, true]; yield 'Laravel client request' => [new LaravelClientRequest($psr7Request), true]; yield 'Laravel client response' => [$laravelClientResponse, false]; diff --git a/tests/Feature/OptimizationTest.php b/tests/Feature/OptimizationTest.php new file mode 100644 index 0000000..d804a94 --- /dev/null +++ b/tests/Feature/OptimizationTest.php @@ -0,0 +1,24 @@ +middleware('log', Middleware::tap($before, $after)) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', + ]); + + expect($log)->toBe(['req1', 'res1', 'req2', 'res2', 'req3', 'res3']); +}); diff --git a/tests/Pest.php b/tests/Pest.php index ec36cc9..b6df0d5 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -1,10 +1,7 @@ extend('toLoadItemsViaRequests', function (array $requests, Generator|array $headers = []) { - $responses = $transactions = $expectedUris = []; + $responses = $expectedUris = []; $responseHeaders = $headers; foreach ($requests as $uri => $fixture) { @@ -48,13 +45,7 @@ $expectedUris[] = $uri; } - $stack = HandlerStack::create(new MockHandler($responses)); - - $stack->push(Middleware::history($transactions)); - - Client::configure(['handler' => $stack]); - - $this->sequence(...require fixture('items.php')); + $transactions = ClientFactory::fake($responses, fn() => $this->sequence(...require fixture('items.php'))); $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); @@ -62,23 +53,18 @@ }); expect()->extend('toFailRequest', function (string $uri) { - $transactions = []; - $responses = [$exception = new RequestException('connection failed', new Request('GET', $uri))]; - $stack = HandlerStack::create(new MockHandler($responses)); - - $stack->push(Middleware::history($transactions)); - - Client::configure(['handler' => $stack]); - - try { - iterator_to_array($this->value); - } catch (Throwable $e) { - expect($e)->toBe($exception); - } + $transactions = ClientFactory::fake($responses, function() use ($exception) { + try { + iterator_to_array($this->value); + } catch (Throwable $e) { + expect($e)->toBe($exception); + } + }); expect($transactions)->toHaveCount(1); + expect((string) $transactions[0]['request']->getUri())->toBe($uri); }); From 71003ec58666f292770510b899c47f6df245044d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 12 Feb 2024 15:45:33 +1000 Subject: [PATCH 085/109] Update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e957f1..bb07ce0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ $lazyCollection = LazyJsonPages::from($source) Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. > [!TIP] -> Do you need to read large JSON with no pagination in a memory-efficient way? Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. +> Need to read large JSON with no pagination in a memory-efficient way? +> Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. ## ๐Ÿ“ฆ Install @@ -293,7 +294,7 @@ LazyJsonPages::from($source) ->middleware('cache_responses', $cacheResponses); ``` -If we a middleware to be added every time we invoke Lazy JSON Pages, we can add a global middleware: +If we need a middleware to be added every time we invoke Lazy JSON Pages, we can add a global middleware: ```php LazyJsonPages::globalMiddleware('fire_events', $fireEvents); From 24887ed3142335899408fa051b7cb93c9f4f6854 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 12 Feb 2024 15:46:18 +1000 Subject: [PATCH 086/109] Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index bb07ce0..16c60e7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Framework-agnostic package to load items from any paginated JSON API into a [Lar > [!TIP] > Need to read large JSON with no pagination in a memory-efficient way? +> > Consider using [๐Ÿผ Lazy JSON](https://github.com/cerbero90/lazy-json) or [๐Ÿงฉ JSON Parser](https://github.com/cerbero90/json-parser) instead. From a0b2d87701904dcbd8b2ddb244b14c93785cb9fe Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 12 Feb 2024 18:31:03 +1000 Subject: [PATCH 087/109] Implement onRequest, onResponse and onError --- README.md | 9 ++ src/LazyJsonPages.php | 72 ++++++++----- src/Services/ClientFactory.php | 111 ++++++++++++++++++--- tests/Feature/OptimizationTest.php | 24 ----- tests/Feature/RequestsOptimizationTest.php | 51 ++++++++++ 5 files changed, 206 insertions(+), 61 deletions(-) delete mode 100644 tests/Feature/OptimizationTest.php create mode 100644 tests/Feature/RequestsOptimizationTest.php diff --git a/README.md b/README.md index 16c60e7..bccffdc 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,15 @@ If we need a middleware to be added every time we invoke Lazy JSON Pages, we can LazyJsonPages::globalMiddleware('fire_events', $fireEvents); ``` +Sometimes writing Guzzle middleware might be cumbersome, alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request, receiving a response or dealing with a transaction error: + +```php +LazyJsonPages::from($source) + ->onRequest(fn(RequestInterface $request) => ...) + ->onResponse(fn(ResponseInterface $response, RequestInterface $request) => ...) + ->onError(fn(Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); +``` + ### ๐Ÿ’ข Errors handling diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 3677ff9..fcc575f 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -12,6 +12,8 @@ use Closure; use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; /** * The Lazy JSON Pages entry-point @@ -23,24 +25,6 @@ final class LazyJsonPages */ private readonly ClientFactory $factory; - /** - * The Guzzle HTTP request options. - * - * @var array - */ - private array $options = [ - RequestOptions::CONNECT_TIMEOUT => 5, - RequestOptions::READ_TIMEOUT => 5, - RequestOptions::TIMEOUT => 5, - ]; - - /** - * The Guzzle client middleware. - * - * @var array - */ - private array $middleware = []; - /** * The raw configuration of the API pagination. * @@ -195,7 +179,7 @@ public function async(int $requests): self */ public function connectionTimeout(float|int $seconds): self { - $this->options[RequestOptions::CONNECT_TIMEOUT] = max(0, $seconds); + $this->factory->option(RequestOptions::CONNECT_TIMEOUT, max(0, $seconds)); return $this; } @@ -205,8 +189,8 @@ public function connectionTimeout(float|int $seconds): self */ public function requestTimeout(float|int $seconds): self { - $this->options[RequestOptions::TIMEOUT] = max(0, $seconds); - $this->options[RequestOptions::READ_TIMEOUT] = max(0, $seconds); + $this->factory->option(RequestOptions::TIMEOUT, max(0, $seconds)); + $this->factory->option(RequestOptions::READ_TIMEOUT, max(0, $seconds)); return $this; } @@ -236,7 +220,43 @@ public function backoff(Closure $callback): self */ public function middleware(string $name, callable $middleware): self { - $this->middleware[$name] = $middleware; + $this->factory->middleware($name, $middleware); + + return $this; + } + + /** + * Handle the sending request. + * + * @param (Closure(RequestInterface): void) $callback + */ + public function onRequest(Closure $callback): self + { + $this->factory->onRequest($callback); + + return $this; + } + + /** + * Handle the received response. + * + * @param (Closure(ResponseInterface, RequestInterface): void) $callback + */ + public function onResponse(Closure $callback): self + { + $this->factory->onResponse($callback); + + return $this; + } + + /** + * Handle a transaction error. + * + * @param (Closure(\Throwable, RequestInterface, ?ResponseInterface): void) $callback + */ + public function onError(Closure $callback): self + { + $this->factory->onError($callback); return $this; } @@ -249,9 +269,11 @@ public function middleware(string $name, callable $middleware): self */ public function collect(string $dot = '*'): LazyCollection { - return new LazyCollection(function() use ($dot) { - $config = new Config(...$this->config, itemsPointer: DotsConverter::toPointer($dot)); - $client = $this->factory->options($this->options)->middleware($this->middleware)->make(); + $this->config['itemsPointer'] = DotsConverter::toPointer($dot); + + return new LazyCollection(function() { + $client = $this->factory->make(); + $config = new Config(...$this->config); $source = new AnySource($this->source, $client); yield from new AnyPagination($source, $client, $config); diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php index 05b4142..fcdfbf4 100644 --- a/src/Services/ClientFactory.php +++ b/src/Services/ClientFactory.php @@ -6,10 +6,14 @@ use Closure; use GuzzleHttp\Client; +use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; +use GuzzleHttp\Promise\Create; use GuzzleHttp\RequestOptions; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; /** * The HTTP client factory. @@ -22,6 +26,9 @@ final class ClientFactory * @var array */ private static array $defaultOptions = [ + RequestOptions::CONNECT_TIMEOUT => 5, + RequestOptions::READ_TIMEOUT => 5, + RequestOptions::TIMEOUT => 5, RequestOptions::STREAM => true, RequestOptions::HEADERS => [ 'Accept' => 'application/json', @@ -46,10 +53,31 @@ final class ClientFactory /** * The local middleware. * - * @var array + * @var array */ private array $middleware = []; + /** + * The callbacks to handle the sending request. + * + * @var Closure[] + */ + private array $onRequestCallbacks = []; + + /** + * The callbacks to handle the received response. + * + * @var Closure[] + */ + private array $onResponseCallbacks = []; + + /** + * The callbacks to handle a transaction error. + * + * @var Closure[] + */ + private array $onErrorCallbacks = []; + /** * Add a global middleware. */ @@ -61,7 +89,7 @@ public static function globalMiddleware(string $name, callable $middleware): voi /** * Fake HTTP requests for testing purposes. * - * @param \Psr\Http\Message\ResponseInterface[]|GuzzleHttp\Exception\RequestException[] $responses + * @param ResponseInterface[]|RequestException[] $responses * @return array> */ public static function fake(array $responses, Closure $callback): array @@ -82,29 +110,88 @@ public static function fake(array $responses, Closure $callback): array } /** - * Set the Guzzle client options. - * - * @param array $options + * Add the given Guzzle client option. */ - public function options(array $options): self + public function option(string $name, mixed $value): self { - $this->options = $options; + $this->options[$name] = $value; return $this; } /** - * Set the Guzzle client middleware. - * - * @param array $middleware + * Add the given Guzzle client middleware. */ - public function middleware(array $middleware): self + public function middleware(string $name, callable $middleware): self { - $this->middleware = $middleware; + $this->middleware[$name] = $middleware; return $this; } + /** + * Add the given callback to handle the sending request. + */ + public function onRequest(Closure $callback): self + { + $this->onRequestCallbacks[] = $callback; + + return $this->tap(); + } + + /** + * Add the middleware to handle a request before and after it is sent. + */ + private function tap(): self + { + $this->middleware['lazy_json_pages_tap'] ??= function (callable $handler): callable { + return function (RequestInterface $request, array $options) use ($handler) { + foreach ($this->onRequestCallbacks as $callback) { + $callback($request); + } + + return $handler($request, $options)->then(function(ResponseInterface $response) use ($request) { + foreach ($this->onResponseCallbacks as $callback) { + $callback($response, $request); + } + + return $response; + }, function(mixed $reason) use ($request) { + $exception = Create::exceptionFor($reason); + $response = $reason instanceof RequestException ? $reason->getResponse() : null; + + foreach ($this->onErrorCallbacks as $callback) { + $callback($exception, $request, $response); + } + + return Create::rejectionFor($reason); + }); + }; + }; + + return $this; + } + + /** + * Add the given callback to handle the received response. + */ + public function onResponse(Closure $callback): self + { + $this->onResponseCallbacks[] = $callback; + + return $this->tap(); + } + + /** + * Add the given callback to handle a transaction error. + */ + public function onError(Closure $callback): self + { + $this->onErrorCallbacks[] = $callback; + + return $this->tap(); + } + /** * Retrieve a configured Guzzle client instance. */ diff --git a/tests/Feature/OptimizationTest.php b/tests/Feature/OptimizationTest.php deleted file mode 100644 index d804a94..0000000 --- a/tests/Feature/OptimizationTest.php +++ /dev/null @@ -1,24 +0,0 @@ -middleware('log', Middleware::tap($before, $after)) - ->totalPages('meta.total_pages') - ->collect('data.*'); - - expect($lazyCollection)->toLoadItemsViaRequests([ - 'https://example.com/api/v1/users' => 'pagination/page1.json', - 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', - 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', - ]); - - expect($log)->toBe(['req1', 'res1', 'req2', 'res2', 'req3', 'res3']); -}); diff --git a/tests/Feature/RequestsOptimizationTest.php b/tests/Feature/RequestsOptimizationTest.php new file mode 100644 index 0000000..6c8f2f9 --- /dev/null +++ b/tests/Feature/RequestsOptimizationTest.php @@ -0,0 +1,51 @@ +middleware('log', Middleware::tap(fn() => $log->push('before'), fn() => $log->push('after'))) + ->onRequest(fn() => $log->push('onRequest')) + ->onResponse(fn() => $log->push('onResponse')) + ->sync() + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', + ]); + + expect($log)->sequence(...[ + 'before', + 'onRequest', + 'after', + 'onResponse', + 'before', + 'onRequest', + 'after', + 'onResponse', + 'before', + 'onRequest', + 'after', + 'onResponse', + ]); +}); + +it('handles transaction errors', function () { + $log = collect(); + + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->middleware('log', Middleware::tap(fn() => $log->push('before'), fn() => $log->push('after'))) + ->onError(fn() => $log->push('onError')) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toFailRequest('https://example.com/api/v1/users'); + + expect($log)->sequence('before', 'after', 'onError'); +}); From ff6fe7f3d043af84b7b20e26b080c77831d5e17b Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 12 Feb 2024 19:11:15 +1000 Subject: [PATCH 088/109] Update readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index bccffdc..3a39442 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,9 @@ LazyJsonPages::globalMiddleware('fire_events', $fireEvents); Sometimes writing Guzzle middleware might be cumbersome, alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request, receiving a response or dealing with a transaction error: ```php +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; + LazyJsonPages::from($source) ->onRequest(fn(RequestInterface $request) => ...) ->onResponse(fn(ResponseInterface $response, RequestInterface $request) => ...) From de83c3b231bde0c2ef82dcb288d5b077c177e3d1 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 13 Feb 2024 19:00:38 +1000 Subject: [PATCH 089/109] Send synchronous HTTP requests by default --- src/Dtos/Config.php | 2 +- src/LazyJsonPages.php | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Dtos/Config.php b/src/Dtos/Config.php index 0c13352..37e8c4a 100644 --- a/src/Dtos/Config.php +++ b/src/Dtos/Config.php @@ -29,7 +29,7 @@ public function __construct( public readonly ?string $offsetKey = null, public readonly bool $hasLinkHeader = false, public readonly ?string $pagination = null, - public readonly int $async = 3, + public readonly int $async = 1, public readonly int $attempts = 3, public readonly ?Closure $backoff = null, ) {} diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index fcc575f..33d27dd 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -156,14 +156,6 @@ public function pagination(string $class): self return $this; } - /** - * Fetch pages synchronously. - */ - public function sync(): self - { - return $this->async(1); - } - /** * Set the maximum number of concurrent async HTTP requests. */ From 06dacebca6772499cda7cb577d4efa65b0d1cce0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 13 Feb 2024 19:00:57 +1000 Subject: [PATCH 090/109] Test async requests --- tests/Feature/RequestsOptimizationTest.php | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/Feature/RequestsOptimizationTest.php b/tests/Feature/RequestsOptimizationTest.php index 6c8f2f9..b7d0832 100644 --- a/tests/Feature/RequestsOptimizationTest.php +++ b/tests/Feature/RequestsOptimizationTest.php @@ -10,7 +10,6 @@ ->middleware('log', Middleware::tap(fn() => $log->push('before'), fn() => $log->push('after'))) ->onRequest(fn() => $log->push('onRequest')) ->onResponse(fn() => $log->push('onResponse')) - ->sync() ->totalPages('meta.total_pages') ->collect('data.*'); @@ -49,3 +48,22 @@ expect($log)->sequence('before', 'after', 'onError'); }); + +it('sends HTTP requests asynchronously', function () { + $log = collect(); + + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->onRequest(fn() => $log->push('sending')) + ->onResponse(fn() => $log->push('sent')) + ->async(3) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?page=3' => 'pagination/page3.json', + ]); + + expect($log)->sequence('sending', 'sent', 'sending', 'sending', 'sent', 'sent'); +}); From e2c724a1997b12fa2af21f170a4fea6dca3196cc Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 13 Feb 2024 19:01:06 +1000 Subject: [PATCH 091/109] Update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 3a39442..108ac98 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,12 @@ If you find yourself implementing the same custom pagination in different projec Paginated APIs differ from each other, so Lazy JSON Pages lets us tweak our HTTP requests specifically for our use case. +By default HTTP requests are sent synchronously. If we want to send more than one request without waiting for the response, we can call the `async()` method and set the number of concurrent requests: + +```php +LazyJsonPages::from($source)->async(requests: 5); +``` + Internally, Lazy JSON Pages uses [Guzzle](https://docs.guzzlephp.org) as its HTTP client. We can customize the client behavior by adding as many [middleware](https://docs.guzzlephp.org/en/stable/handlers-and-middleware.html#middleware) as we need: ```php From 0ca3adc10b9afd9b5dc9be6d61206a2146326556 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Thu, 15 Feb 2024 19:17:07 +1000 Subject: [PATCH 092/109] Make key read-only --- src/Exceptions/InvalidKeyException.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Exceptions/InvalidKeyException.php b/src/Exceptions/InvalidKeyException.php index 536b778..3422ada 100644 --- a/src/Exceptions/InvalidKeyException.php +++ b/src/Exceptions/InvalidKeyException.php @@ -10,7 +10,7 @@ class InvalidKeyException extends LazyJsonPagesException /** * Instantiate the class. */ - public function __construct(public string $key) + public function __construct(public readonly string $key) { parent::__construct("The key [{$key}] does not contain a valid value."); } From 00ddf7ebcb78dd287cc4262f6dff7c5eaaab9283 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 16 Feb 2024 21:50:24 +1000 Subject: [PATCH 093/109] Introduce the tap middleware --- src/LazyJsonPages.php | 43 ++++--- src/Middleware/Tap.php | 99 ++++++++++++++++ .../LazyJsonPagesServiceProvider.php | 29 +++-- src/Services/ClientFactory.php | 106 ++++++++---------- src/Services/TapCallbacks.php | 96 ++++++++++++++++ 5 files changed, 287 insertions(+), 86 deletions(-) create mode 100644 src/Middleware/Tap.php create mode 100644 src/Services/TapCallbacks.php diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 33d27dd..b76cdda 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -12,8 +12,9 @@ use Closure; use GuzzleHttp\RequestOptions; use Illuminate\Support\LazyCollection; -use Psr\Http\Message\RequestInterface; -use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\RequestInterface as Request; +use Psr\Http\Message\ResponseInterface as Response; +use Throwable; /** * The Lazy JSON Pages entry-point @@ -23,7 +24,7 @@ final class LazyJsonPages /** * The HTTP client factory. */ - private readonly ClientFactory $factory; + private readonly ClientFactory $client; /** * The raw configuration of the API pagination. @@ -53,7 +54,7 @@ public static function from(mixed $source): self */ public function __construct(private readonly mixed $source) { - $this->factory = new ClientFactory(); + $this->client = new ClientFactory(); } /** @@ -171,7 +172,7 @@ public function async(int $requests): self */ public function connectionTimeout(float|int $seconds): self { - $this->factory->option(RequestOptions::CONNECT_TIMEOUT, max(0, $seconds)); + $this->client->config(RequestOptions::CONNECT_TIMEOUT, max(0, $seconds)); return $this; } @@ -181,8 +182,8 @@ public function connectionTimeout(float|int $seconds): self */ public function requestTimeout(float|int $seconds): self { - $this->factory->option(RequestOptions::TIMEOUT, max(0, $seconds)); - $this->factory->option(RequestOptions::READ_TIMEOUT, max(0, $seconds)); + $this->client->config(RequestOptions::TIMEOUT, max(0, $seconds)); + $this->client->config(RequestOptions::READ_TIMEOUT, max(0, $seconds)); return $this; } @@ -212,7 +213,7 @@ public function backoff(Closure $callback): self */ public function middleware(string $name, callable $middleware): self { - $this->factory->middleware($name, $middleware); + $this->client->middleware($name, $middleware); return $this; } @@ -220,11 +221,11 @@ public function middleware(string $name, callable $middleware): self /** * Handle the sending request. * - * @param (Closure(RequestInterface): void) $callback + * @param Closure(Request $request, array $config): void $callback */ public function onRequest(Closure $callback): self { - $this->factory->onRequest($callback); + $this->client->onRequest($callback); return $this; } @@ -232,11 +233,11 @@ public function onRequest(Closure $callback): self /** * Handle the received response. * - * @param (Closure(ResponseInterface, RequestInterface): void) $callback + * @param Closure(Response $response, Request $request, array $config): void $callback */ public function onResponse(Closure $callback): self { - $this->factory->onResponse($callback); + $this->client->onResponse($callback); return $this; } @@ -244,11 +245,23 @@ public function onResponse(Closure $callback): self /** * Handle a transaction error. * - * @param (Closure(\Throwable, RequestInterface, ?ResponseInterface): void) $callback + * @param Closure(Throwable $e, Request $request, ?Response $response, array $config): void $callback */ public function onError(Closure $callback): self { - $this->factory->onError($callback); + $this->client->onError($callback); + + return $this; + } + + /** + * Throttle the requests to respect rate limiting. + */ + public function throttle(int $requests, int $perSeconds = 0, int $perMinutes = 0, int $perHours = 0): self + { + $seconds = max(0, $perSeconds + $perMinutes * 60 + $perHours * 3600); + + $this->client->throttle($requests, $seconds); return $this; } @@ -264,7 +277,7 @@ public function collect(string $dot = '*'): LazyCollection $this->config['itemsPointer'] = DotsConverter::toPointer($dot); return new LazyCollection(function() { - $client = $this->factory->make(); + $client = $this->client->make(); $config = new Config(...$this->config); $source = new AnySource($this->source, $client); diff --git a/src/Middleware/Tap.php b/src/Middleware/Tap.php new file mode 100644 index 0000000..b5b5480 --- /dev/null +++ b/src/Middleware/Tap.php @@ -0,0 +1,99 @@ + + */ + private array $config; + + /** + * Instantiate the class statically to tap once. + */ + public static function once(?Closure $onRequest = null, ?Closure $onResponse = null, ?Closure $onError = null): self + { + $callbacks = new TapCallbacks(); + $onRequest && $callbacks->onRequest($onRequest); + $onResponse && $callbacks->onResponse($onResponse); + $onError && $callbacks->onError($onError); + + return new self($callbacks); + } + + /** + * Instantiate the class. + */ + public function __construct(private readonly TapCallbacks $callbacks) + { + } + + /** + * Handle an HTTP request before and after it is sent. + * + * @param callable(RequestInterface, array): PromiseInterface $handler + */ + public function __invoke(callable $handler): Closure + { + return function (RequestInterface $request, array $config) use ($handler) { + $this->request = $request; + $this->config = $config; + + foreach ($this->callbacks->onRequestCallbacks() as $callback) { + $callback($request, $config); + } + + return $handler($request, $config) + ->then($this->handleResponse(...)) + ->otherwise($this->handleError(...)); + }; + } + + /** + * Handle the given response. + */ + private function handleResponse(ResponseInterface $response): ResponseInterface + { + foreach ($this->callbacks->onResponseCallbacks() as $callback) { + $callback($response, $this->request, $this->config); + } + + return $response; + } + + /** + * Handle the given transaction error. + */ + private function handleError(mixed $reason): PromiseInterface + { + $exception = Create::exceptionFor($reason); + $response = $reason instanceof RequestException ? $reason->getResponse() : null; + + foreach ($this->callbacks->onErrorCallbacks() as $callback) { + $callback($exception, $this->request, $response, $this->config); + } + + return Create::rejectionFor($reason); + } +} diff --git a/src/Providers/LazyJsonPagesServiceProvider.php b/src/Providers/LazyJsonPagesServiceProvider.php index 9c7fff8..01a3c46 100644 --- a/src/Providers/LazyJsonPagesServiceProvider.php +++ b/src/Providers/LazyJsonPagesServiceProvider.php @@ -5,8 +5,7 @@ namespace Cerbero\LazyJsonPages\Providers; use Cerbero\LazyJsonPages\LazyJsonPages; -use GuzzleHttp\Middleware; -use GuzzleHttp\Promise\PromiseInterface; +use Cerbero\LazyJsonPages\Middleware\Tap; use Illuminate\Http\Client\Events\ConnectionFailed; use Illuminate\Http\Client\Events\RequestSending; use Illuminate\Http\Client\Events\ResponseReceived; @@ -15,6 +14,7 @@ use Illuminate\Support\ServiceProvider; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; +use Throwable; /** * The service provider to integrate with Laravel. @@ -26,27 +26,32 @@ final class LazyJsonPagesServiceProvider extends ServiceProvider */ public function boot(): void { - LazyJsonPages::globalMiddleware('laravel_events', Middleware::tap($this->sending(...), $this->sent(...))); + $fireEvents = Tap::once($this->onRequest(...), $this->onResponse(...), $this->onError(...)); + + LazyJsonPages::globalMiddleware('laravel_events', $fireEvents); } /** * Handle HTTP requests before they are sent. */ - private function sending(RequestInterface $request): void + private function onRequest(RequestInterface $request): void { - event(new RequestSending(new Request($request))); + $this->app['events']->dispatch(new RequestSending(new Request($request))); } /** - * Handle HTTP requests after they are sent. + * Handle HTTP responses after they are received. */ - private function sent(RequestInterface $request, array $options, PromiseInterface $promise): void + private function onResponse(ResponseInterface $response, RequestInterface $request): void { - $clientRequest = new Request($request); + $this->app['events']->dispatch(new ResponseReceived(new Request($request), new Response($response))); + } - $promise->then( - fn(ResponseInterface $response) => event(new ResponseReceived($clientRequest, new Response($response))), - fn() => event(new ConnectionFailed($clientRequest)), - ); + /** + * Handle a transaction error. + */ + private function onError(Throwable $e, RequestInterface $request): void + { + $this->app['events']->dispatch(new ConnectionFailed(new Request($request))); } } diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php index fcdfbf4..2e6d08a 100644 --- a/src/Services/ClientFactory.php +++ b/src/Services/ClientFactory.php @@ -4,15 +4,15 @@ namespace Cerbero\LazyJsonPages\Services; +use Cerbero\LazyJsonPages\Middleware\Tap; +use Cerbero\LazyJsonPages\Services\TapCallbacks; use Closure; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; -use GuzzleHttp\Promise\Create; use GuzzleHttp\RequestOptions; -use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; /** @@ -21,11 +21,11 @@ final class ClientFactory { /** - * The default options. + * The default client configuration. * * @var array */ - private static array $defaultOptions = [ + private static array $defaultConfig = [ RequestOptions::CONNECT_TIMEOUT => 5, RequestOptions::READ_TIMEOUT => 5, RequestOptions::TIMEOUT => 5, @@ -44,11 +44,16 @@ final class ClientFactory private static array $globalMiddleware = []; /** - * The custom options. + * The tap middleware callbacks. + */ + private readonly TapCallbacks $tapCallbacks; + + /** + * The custom client configuration. * * @var array */ - private array $options = []; + private array $config = []; /** * The local middleware. @@ -58,25 +63,11 @@ final class ClientFactory private array $middleware = []; /** - * The callbacks to handle the sending request. - * - * @var Closure[] - */ - private array $onRequestCallbacks = []; - - /** - * The callbacks to handle the received response. - * - * @var Closure[] - */ - private array $onResponseCallbacks = []; - - /** - * The callbacks to handle a transaction error. + * The requests throttling. * - * @var Closure[] + * @var array */ - private array $onErrorCallbacks = []; + private array $throttling = []; /** * Add a global middleware. @@ -100,21 +91,29 @@ public static function fake(array $responses, Closure $callback): array $handler->push(Middleware::history($transactions)); - self::$defaultOptions['handler'] = $handler; + self::$defaultConfig['handler'] = $handler; $callback(); - unset(self::$defaultOptions['handler']); + unset(self::$defaultConfig['handler']); return $transactions; } /** - * Add the given Guzzle client option. + * Instantiate the class. + */ + public function __construct() + { + $this->tapCallbacks = new TapCallbacks(); + } + + /** + * Add the given option to the Guzzle client configuration. */ - public function option(string $name, mixed $value): self + public function config(string $name, mixed $value): self { - $this->options[$name] = $value; + $this->config[$name] = $value; return $this; } @@ -134,7 +133,7 @@ public function middleware(string $name, callable $middleware): self */ public function onRequest(Closure $callback): self { - $this->onRequestCallbacks[] = $callback; + $this->tapCallbacks->onRequest($callback); return $this->tap(); } @@ -144,30 +143,7 @@ public function onRequest(Closure $callback): self */ private function tap(): self { - $this->middleware['lazy_json_pages_tap'] ??= function (callable $handler): callable { - return function (RequestInterface $request, array $options) use ($handler) { - foreach ($this->onRequestCallbacks as $callback) { - $callback($request); - } - - return $handler($request, $options)->then(function(ResponseInterface $response) use ($request) { - foreach ($this->onResponseCallbacks as $callback) { - $callback($response, $request); - } - - return $response; - }, function(mixed $reason) use ($request) { - $exception = Create::exceptionFor($reason); - $response = $reason instanceof RequestException ? $reason->getResponse() : null; - - foreach ($this->onErrorCallbacks as $callback) { - $callback($exception, $request, $response); - } - - return Create::rejectionFor($reason); - }); - }; - }; + $this->middleware['lazy_json_pages_tap'] ??= new Tap($this->tapCallbacks); return $this; } @@ -177,7 +153,7 @@ private function tap(): self */ public function onResponse(Closure $callback): self { - $this->onResponseCallbacks[] = $callback; + $this->tapCallbacks->onResponse($callback); return $this->tap(); } @@ -187,23 +163,35 @@ public function onResponse(Closure $callback): self */ public function onError(Closure $callback): self { - $this->onErrorCallbacks[] = $callback; + $this->tapCallbacks->onError($callback); return $this->tap(); } + /** + * Throttle the requests to respect rate limiting. + */ + public function throttle(int $requests, int $seconds): self + { + $this->throttling[$seconds] = $requests; + + // $this->middleware['lazy_json_pages_throttle'] ??= Tap::once(); + + return $this; + } + /** * Retrieve a configured Guzzle client instance. */ public function make(): Client { - $options = array_replace_recursive(self::$defaultOptions, $this->options); - $options['handler'] ??= HandlerStack::create(); + $config = array_replace_recursive(self::$defaultConfig, $this->config); + $config['handler'] ??= HandlerStack::create(); foreach ([...self::$globalMiddleware, ...$this->middleware] as $name => $middleware) { - $options['handler']->push($middleware, $name); + $config['handler']->push($middleware, $name); } - return new Client($options); + return new Client($config); } } diff --git a/src/Services/TapCallbacks.php b/src/Services/TapCallbacks.php new file mode 100644 index 0000000..f65802a --- /dev/null +++ b/src/Services/TapCallbacks.php @@ -0,0 +1,96 @@ +onRequestCallbacks[] = $callback; + + return $this; + } + + /** + * Add the given callback to handle the received response. + */ + public function onResponse(Closure $callback): self + { + $this->onResponseCallbacks[] = $callback; + + return $this; + } + + /** + * Add the given callback to handle a transaction error. + */ + public function onError(Closure $callback): self + { + $this->onErrorCallbacks[] = $callback; + + return $this; + } + + /** + * Retrieve the callbacks to handle the sending request. + * + * @return Closure[] + */ + public function onRequestCallbacks(): array + { + return $this->onRequestCallbacks; + } + + /** + * Retrieve the callbacks to handle the received response. + * + * @return Closure[] + */ + public function onResponseCallbacks(): array + { + return $this->onResponseCallbacks; + } + + /** + * Retrieve the callbacks to handle a transaction error. + * + * @return Closure[] + */ + public function onErrorCallbacks(): array + { + return $this->onErrorCallbacks; + } +} From 4d836f1db8e08e24bf10a02e14e8d8a50772e2ad Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sun, 18 Feb 2024 01:11:57 +1000 Subject: [PATCH 094/109] Set the HTTP client for sources --- src/LazyJsonPages.php | 2 +- src/Sources/AnySource.php | 4 ++-- src/Sources/CustomSource.php | 4 ++-- src/Sources/Source.php | 16 +++++++++++++++- tests/Feature/Datasets.php | 3 +-- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index b76cdda..da76e78 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -279,7 +279,7 @@ public function collect(string $dot = '*'): LazyCollection return new LazyCollection(function() { $client = $this->client->make(); $config = new Config(...$this->config); - $source = new AnySource($this->source, $client); + $source = (new AnySource($this->source))->setClient($client); yield from new AnyPagination($source, $client, $config); }); diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index c37769b..9cf4614 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -55,10 +55,10 @@ protected function matchingSource(): Source } foreach ($this->supportedSources as $class) { - $source = new $class($this->source, $this->client); + $source = new $class($this->source); if ($source->matches()) { - return $this->matchingSource = $source; + return $this->matchingSource = $source->setClient($this->client); } } diff --git a/src/Sources/CustomSource.php b/src/Sources/CustomSource.php index 3970655..48cc539 100644 --- a/src/Sources/CustomSource.php +++ b/src/Sources/CustomSource.php @@ -27,7 +27,7 @@ public function matches(): bool */ public function request(): RequestInterface { - return $this->source->request(); + return $this->source->setClient($this->client)->request(); } /** @@ -37,6 +37,6 @@ public function request(): RequestInterface */ public function response(): ResponseInterface { - return $this->source->response(); + return $this->source->setClient($this->client)->response(); } } diff --git a/src/Sources/Source.php b/src/Sources/Source.php index d261817..4f4f677 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -13,6 +13,11 @@ */ abstract class Source { + /** + * The HTTP client. + */ + protected readonly Client $client; + /** * Retrieve the HTTP request. */ @@ -30,7 +35,6 @@ abstract public function response(): ResponseInterface; */ final public function __construct( protected readonly mixed $source, - protected readonly Client $client, ) {} /** @@ -40,4 +44,14 @@ public function matches(): bool { return true; } + + /** + * Set the HTTP client. + */ + public function setClient(Client $client): static + { + $this->client ??= $client; + + return $this; + } } diff --git a/tests/Feature/Datasets.php b/tests/Feature/Datasets.php index ed4e439..45ad9b6 100644 --- a/tests/Feature/Datasets.php +++ b/tests/Feature/Datasets.php @@ -2,7 +2,6 @@ use Cerbero\LazyJsonPages\LazyJsonPages; use Cerbero\LazyJsonPages\Paginations\TotalPagesAwarePagination; -use Cerbero\LazyJsonPages\Services\ClientFactory; use Cerbero\LazyJsonPages\Sources\CustomSourceSample; use GuzzleHttp\Psr7\Request as Psr7Request; use GuzzleHttp\Psr7\Response as Psr7Response; @@ -19,7 +18,7 @@ $laravelClientResponse = new LaravelClientResponse($psr7Response); $laravelClientResponse->transferStats = new TransferStats($psr7Request, $psr7Response); - yield 'user-defined source' => [new CustomSourceSample(null, (new ClientFactory())->make()), false]; + yield 'user-defined source' => [new CustomSourceSample(null), false]; yield 'endpoint' => [$uri, true]; yield 'Laravel client request' => [new LaravelClientRequest($psr7Request), true]; yield 'Laravel client response' => [$laravelClientResponse, false]; From c9359d89cd8ebc0761fd775bbbec8d76bbecc0b2 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Sat, 11 May 2024 10:08:46 +0700 Subject: [PATCH 095/109] Update comment --- src/Paginations/AnyPagination.php | 4 ++-- src/Paginations/CursorAwarePagination.php | 2 +- src/Paginations/CustomPagination.php | 2 +- src/Paginations/LastPageAwarePagination.php | 2 +- src/Paginations/LinkHeaderAwarePagination.php | 2 +- src/Paginations/Pagination.php | 2 +- src/Paginations/TotalItemsAwarePagination.php | 2 +- src/Paginations/TotalPagesAwarePagination.php | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Paginations/AnyPagination.php b/src/Paginations/AnyPagination.php index 0d31c9b..1a5371f 100644 --- a/src/Paginations/AnyPagination.php +++ b/src/Paginations/AnyPagination.php @@ -34,8 +34,8 @@ class AnyPagination extends Pagination */ public function getIterator(): Traversable { - // yield only items and not their related index to ensure indexes continuity - // otherwise the actual indexes always start from 0 on every page. + // yield only items and not indexes to ensure incremental indexes + // otherwise the actual indexes always start from 0 on every page foreach ($this->matchingPagination() as $item) { yield $item; } diff --git a/src/Paginations/CursorAwarePagination.php b/src/Paginations/CursorAwarePagination.php index 19b9de6..2448b14 100644 --- a/src/Paginations/CursorAwarePagination.php +++ b/src/Paginations/CursorAwarePagination.php @@ -16,7 +16,7 @@ class CursorAwarePagination extends Pagination use YieldsItemsByCursor; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/CustomPagination.php b/src/Paginations/CustomPagination.php index 82b6983..0e631b8 100644 --- a/src/Paginations/CustomPagination.php +++ b/src/Paginations/CustomPagination.php @@ -13,7 +13,7 @@ class CustomPagination extends Pagination { /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/LastPageAwarePagination.php b/src/Paginations/LastPageAwarePagination.php index bd259b0..94a6224 100644 --- a/src/Paginations/LastPageAwarePagination.php +++ b/src/Paginations/LastPageAwarePagination.php @@ -15,7 +15,7 @@ class LastPageAwarePagination extends Pagination use YieldsItemsByLength; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/LinkHeaderAwarePagination.php b/src/Paginations/LinkHeaderAwarePagination.php index a380b50..3e0a4c0 100644 --- a/src/Paginations/LinkHeaderAwarePagination.php +++ b/src/Paginations/LinkHeaderAwarePagination.php @@ -25,7 +25,7 @@ class LinkHeaderAwarePagination extends Pagination public const FORMAT = '~<\s*(?[^\s>]+)\s*>.*?"\s*(?[^\s"]+)\s*"~'; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 1d07c21..3d8e490 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -47,7 +47,7 @@ final public function __construct( } /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 049b269..2241347 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -15,7 +15,7 @@ class TotalItemsAwarePagination extends Pagination use YieldsItemsByLength; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index d216ac2..824aae9 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -15,7 +15,7 @@ class TotalPagesAwarePagination extends Pagination use YieldsItemsByLength; /** - * Determine whether the configuration matches this pagination. + * Determine whether this pagination matches the configuration. */ public function matches(): bool { From 200d0da17d14fb6a5efac6ed055b1659634c48e0 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 6 Sep 2024 13:29:36 -0300 Subject: [PATCH 096/109] Add requests throttling --- src/Concerns/RespectsRateLimits.php | 21 +++++ src/Concerns/RetriesHttpRequests.php | 62 ++++---------- src/Concerns/SendsAsyncRequests.php | 85 +++++++++++++------ src/Concerns/YieldsItemsByCursor.php | 21 +++-- src/Concerns/YieldsItemsByLength.php | 31 ++----- src/{Dtos => Data}/Config.php | 23 ++++- src/Data/RateLimit.php | 76 +++++++++++++++++ src/Exceptions/OutOfAttemptsException.php | 16 ++-- .../UnsupportedPaginationException.php | 2 +- src/LazyJsonPages.php | 65 +++++++------- src/Paginations/Pagination.php | 2 +- src/Services/Book.php | 12 ++- src/Services/ClientFactory.php | 41 +++++---- src/Services/RateLimits.php | 74 ++++++++++++++++ 14 files changed, 360 insertions(+), 171 deletions(-) create mode 100644 src/Concerns/RespectsRateLimits.php rename src/{Dtos => Data}/Config.php (50%) create mode 100644 src/Data/RateLimit.php create mode 100644 src/Services/RateLimits.php diff --git a/src/Concerns/RespectsRateLimits.php b/src/Concerns/RespectsRateLimits.php new file mode 100644 index 0000000..6b3b000 --- /dev/null +++ b/src/Concerns/RespectsRateLimits.php @@ -0,0 +1,21 @@ +config->rateLimits?->resetAt() ?? 0) { + time_sleep_until($timestamp); + } + } +} diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php index bbba639..d98b9a4 100644 --- a/src/Concerns/RetriesHttpRequests.php +++ b/src/Concerns/RetriesHttpRequests.php @@ -5,44 +5,46 @@ namespace Cerbero\LazyJsonPages\Concerns; use Cerbero\LazyJsonPages\Exceptions\OutOfAttemptsException; -use Closure; use Generator; use GuzzleHttp\Exception\TransferException; -use Illuminate\Support\Sleep; +use Illuminate\Support\LazyCollection; /** * The trait to retry HTTP requests when they fail. * + * @property-read \Cerbero\LazyJsonPages\Data\Config $config + * @property-read \Cerbero\LazyJsonPages\Services\Book $book */ trait RetriesHttpRequests { /** - * Retry to return HTTP responses from the given callback. - * - * @template TReturn + * Retry to yield HTTP responses from the given callback. * - * @param (Closure(Outcome): TReturn) $callback - * @return TReturn + * @param callable $callback + * @return Generator */ - protected function retry(Closure $callback): mixed + protected function retry(callable $callback): Generator { $attempt = 0; $remainingAttempts = $this->config->attempts; do { - $attempt++; - $remainingAttempts--; + $failed = false; + ++$attempt; + --$remainingAttempts; try { - return $callback(); + yield from $callback(); } catch (TransferException $e) { + $failed = true; + if ($remainingAttempts > 0) { $this->backoff($attempt); } else { $this->outOfAttempts($e); } } - } while ($remainingAttempts > 0); + } while ($failed && $remainingAttempts > 0); } /** @@ -52,7 +54,7 @@ protected function backoff(int $attempt): void { $backoff = $this->config->backoff ?: fn(int $attempt) => $attempt ** 2 * 100; - Sleep::for($backoff($attempt) * 1000)->microseconds(); + usleep($backoff($attempt) * 1000); } /** @@ -60,40 +62,10 @@ protected function backoff(int $attempt): void */ protected function outOfAttempts(TransferException $e): never { - throw new OutOfAttemptsException($e, $this->book->pullFailedPages(), function () { + throw new OutOfAttemptsException($e, $this->book->pullFailedPages(), new LazyCollection(function () { foreach ($this->book->pullPages() as $page) { yield from $this->yieldItemsFrom($page); } - }); - } - - /** - * Retry to yield HTTP responses from the given callback. - * - * @param callable $callback - * @return Generator - */ - protected function retryYielding(callable $callback): Generator - { - $attempt = 0; - $remainingAttempts = $this->config->attempts; - - do { - $failed = false; - $attempt++; - $remainingAttempts--; - - try { - yield from $callback(); - } catch (TransferException $e) { - $failed = true; - - if ($remainingAttempts > 0) { - $this->backoff($attempt); - } else { - $this->outOfAttempts($e); - } - } - } while ($failed && $remainingAttempts > 0); + })); } } diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index cdd11ae..273ff7a 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -6,64 +6,95 @@ use Generator; use GuzzleHttp\Pool; -use Illuminate\Support\LazyCollection; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; -use Psr\Http\Message\UriInterface; use Throwable; -use Traversable; /** * The trait to send asynchronous HTTP requests. * + * @property-read \Cerbero\LazyJsonPages\Sources\AnySource $source + * @property-read \Cerbero\LazyJsonPages\Services\Book $book */ trait SendsAsyncRequests { + use RespectsRateLimits; use RetriesHttpRequests; /** - * Fetch pages by performing asynchronous HTTP calls. + * Fetch pages by sending asynchronous HTTP requests. * - * @param LazyCollection> $chunkedPages - * @return Traversable + * @return Generator */ - protected function fetchPagesAsynchronously(LazyCollection $chunkedPages, UriInterface $uri): Traversable + protected function fetchPagesAsynchronously(int $totalPages): Generator { - foreach ($chunkedPages as $pages) { - $this->retry(fn() => $this->pool($uri, $pages->all())->promise()->wait()); + $request = clone $this->source->request(); + $fromPage = $this->config->firstPage + 1; + $toPage = $this->config->firstPage == 0 ? $totalPages - 1 : $totalPages; - yield from $this->book->pullPages(); - } + yield from $this->retry(function() use ($request, &$fromPage, $toPage) { + foreach ($this->chunkRequestsBetweenPages($request, $fromPage, $toPage) as $requests) { + yield from $this->pool($requests); + } + }); } /** - * Retrieve a pool of asynchronous requests. + * Retrieve requests for the given pages in chunks. * - * @param array $pages + * @return Generator> */ - protected function pool(UriInterface $uri, array $pages): Pool + protected function chunkRequestsBetweenPages(RequestInterface $request, int &$fromPage, int $toPage): Generator { - return new Pool($this->client, $this->yieldRequests($uri, $pages), [ - 'concurrency' => $this->config->async, - 'fulfilled' => fn(ResponseInterface $response, int $page) => $this->book->addPage($page, $response), - 'rejected' => fn(Throwable $e, int $page) => $this->book->addFailedPage($page) && throw $e, - ]); + while ($fromPage <= $toPage) { + yield $this->yieldRequestsBetweenPages($request, $fromPage, $toPage); + + $this->respectRateLimits(); + } } /** - * Retrieve a generator yielding the HTTP requests for the given pages. + * Yield the requests between the given pages. * - * @param array $pages * @return Generator */ - protected function yieldRequests(UriInterface $uri, array $pages): Generator + protected function yieldRequestsBetweenPages(RequestInterface $request, int &$fromPage, int $toPage): Generator { - /** @var RequestInterface $request */ - $request = clone $this->source->request(); - $pages = $this->book->pullFailedPages() ?: $pages; + $chunkSize = min($this->config->async, $this->config->rateLimits?->threshold() ?? INF); + + for ($i = 0; $i < $chunkSize && $fromPage <= $toPage; $i++) { + $page = $this->book->pullFailedPage() ?? $fromPage++; - foreach ($pages as $page) { - yield $page => $request->withUri($this->uriForPage($uri, (string) $page)); + yield $page => $request->withUri($this->uriForPage($request->getUri(), (string) $page)); } } + + /** + * Send a pool of asynchronous requests. + * + * @param Generator $requests + * @return Generator + * @throws Throwable + */ + protected function pool(Generator $requests): Generator + { + $exception = null; + + $config = [ + 'concurrency' => $this->config->async, + 'fulfilled' => fn(ResponseInterface $response, int $page) => $this->book->addPage($page, $response), + 'rejected' => function(Throwable $e, int $page) use (&$exception) { + $this->book->addFailedPage($page); + $exception = $e; + }, + ]; + + (new Pool($this->client, $requests, $config))->promise()->wait(); + + if (isset($exception)) { + throw $exception; + } + + yield from $this->book->pullPages(); + } } diff --git a/src/Concerns/YieldsItemsByCursor.php b/src/Concerns/YieldsItemsByCursor.php index ec8c1c2..2ae9a8f 100644 --- a/src/Concerns/YieldsItemsByCursor.php +++ b/src/Concerns/YieldsItemsByCursor.php @@ -13,23 +13,30 @@ */ trait YieldsItemsByCursor { + use RespectsRateLimits; + use RetriesHttpRequests; + /** * Yield the paginated items by following the cursor of each page. * - * @param (Closure(ResponseInterface): Generator) $callback + * @param Closure(ResponseInterface): Generator $callback * @return Generator */ protected function yieldItemsByCursor(Closure $callback): Generator { + $request = clone $this->source->request(); + yield from $generator = $callback($this->source->pullResponse()); - $request = clone $this->source->request(); + yield from $this->retry(function() use ($callback, $request, $generator) { + while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { + $this->respectRateLimits(); - while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { - $uri = $this->uriForPage($request->getUri(), (string) $cursor); - $response = $this->client->send($request->withUri($uri)); + $uri = $this->uriForPage($request->getUri(), (string) $cursor); + $response = $this->client->send($request->withUri($uri)); - yield from $generator = $callback($response); - } + yield from $generator = $callback($response); + } + }); } } diff --git a/src/Concerns/YieldsItemsByLength.php b/src/Concerns/YieldsItemsByLength.php index 50a39b9..188e995 100644 --- a/src/Concerns/YieldsItemsByLength.php +++ b/src/Concerns/YieldsItemsByLength.php @@ -7,7 +7,6 @@ use Cerbero\LazyJsonPages\Exceptions\InvalidKeyException; use Closure; use Generator; -use Illuminate\Support\LazyCollection; use Psr\Http\Message\ResponseInterface; /** @@ -20,10 +19,10 @@ trait YieldsItemsByLength /** * Yield paginated items until the page resolved from the given key is reached. * - * @param (Closure(int): int) $callback - * @return Generator + * @param ?Closure(int): int $callback + * @return Generator */ - protected function yieldItemsUntilKey(string $key, Closure $callback = null): Generator + protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): Generator { yield from $this->yieldItemsUntilPage(function(ResponseInterface $response) use ($key, $callback) { yield from $generator = $this->yieldItemsAndGetKey($response, $key); @@ -39,33 +38,15 @@ protected function yieldItemsUntilKey(string $key, Closure $callback = null): Ge /** * Yield paginated items until the resolved page is reached. * - * @param (Closure(ResponseInterface): Generator) $callback + * @param Closure(ResponseInterface): Generator $callback * @return Generator */ protected function yieldItemsUntilPage(Closure $callback): Generator { yield from $generator = $callback($this->source->pullResponse()); - $uri = $this->source->request()->getUri(); - $chunkedPages = $this->chunkPages($generator->getReturn()); - - foreach ($this->fetchPagesAsynchronously($chunkedPages, $uri) as $page) { - yield from $this->yieldItemsFrom($page); + foreach ($this->fetchPagesAsynchronously($generator->getReturn()) as $response) { + yield from $this->yieldItemsFrom($response); } } - - /** - * Retrieve the given pages in chunks. - * - * @return LazyCollection> - */ - protected function chunkPages(int $pages): LazyCollection - { - $firstPage = $this->config->firstPage + 1; - $lastPage = $this->config->firstPage == 0 ? $pages - 1 : $pages; - - return $firstPage > $lastPage - ? LazyCollection::empty() - : LazyCollection::range($firstPage, $lastPage)->chunk($this->config->async); - } } diff --git a/src/Dtos/Config.php b/src/Data/Config.php similarity index 50% rename from src/Dtos/Config.php rename to src/Data/Config.php index 37e8c4a..57fc8bc 100644 --- a/src/Dtos/Config.php +++ b/src/Data/Config.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Cerbero\LazyJsonPages\Dtos; +namespace Cerbero\LazyJsonPages\Data; use Cerbero\LazyJsonPages\Paginations\Pagination; +use Cerbero\LazyJsonPages\Services\RateLimits; use Closure; /** @@ -14,6 +15,25 @@ */ final class Config { + /** + * The configuration options. + */ + public const OPTION_PAGE_NAME = 'pageName'; + public const OPTION_PAGE_IN_PATH = 'pageInPath'; + public const OPTION_FIRST_PAGE = 'firstPage'; + public const OPTION_TOTAL_PAGES_KEY = 'totalPagesKey'; + public const OPTION_TOTAL_ITEMS_KEY = 'totalItemsKey'; + public const OPTION_CURSOR_KEY = 'cursorKey'; + public const OPTION_LAST_PAGE_KEY = 'lastPageKey'; + public const OPTION_OFFSET_KEY = 'offsetKey'; + public const OPTION_HAS_LINK_HEADER = 'hasLinkHeader'; + public const OPTION_PAGINATION = 'pagination'; + public const OPTION_RATE_LIMITS = 'rateLimits'; + public const OPTION_ASYNC = 'async'; + public const OPTION_ATTEMPTS = 'attempts'; + public const OPTION_BACKOFF = 'backoff'; + public const OPTION_ITEMS_POINTER = 'itemsPointer'; + /** * Instantiate the class. */ @@ -29,6 +49,7 @@ public function __construct( public readonly ?string $offsetKey = null, public readonly bool $hasLinkHeader = false, public readonly ?string $pagination = null, + public readonly ?RateLimits $rateLimits = null, public readonly int $async = 1, public readonly int $attempts = 3, public readonly ?Closure $backoff = null, diff --git a/src/Data/RateLimit.php b/src/Data/RateLimit.php new file mode 100644 index 0000000..99ecb29 --- /dev/null +++ b/src/Data/RateLimit.php @@ -0,0 +1,76 @@ +hits++; + $this->resetsAt ??= microtime(true) + $this->perSeconds; + + return $this; + } + + /** + * Retrieve the number of requests allowed before this rate limit resets. + */ + public function threshold(): int + { + return $this->requests - $this->hits; + } + + /** + * Determine whether this rate limit was reached. + */ + public function wasReached(): bool + { + return $this->hits == $this->requests; + } + + /** + * Retrieve the timestamp when this rate limit resets. + */ + public function resetsAt(): float + { + return $this->resetsAt; + } + + /** + * Reset this rate limit. + */ + public function reset(): self + { + $this->hits = 0; + $this->resetsAt = null; + + return $this; + } +} diff --git a/src/Exceptions/OutOfAttemptsException.php b/src/Exceptions/OutOfAttemptsException.php index ecfcf93..b6b0aa6 100644 --- a/src/Exceptions/OutOfAttemptsException.php +++ b/src/Exceptions/OutOfAttemptsException.php @@ -11,23 +11,17 @@ */ class OutOfAttemptsException extends LazyJsonPagesException { - /** - * The paginated items loaded before the failure. - * - * @var LazyCollection - */ - public readonly LazyCollection $items; - /** * Instantiate the class. * * @param array $failedPages * @param (Closure(): Generator) $items */ - public function __construct(TransferException $e, public readonly array $failedPages, Closure $items) - { - $this->items = new LazyCollection($items); - + public function __construct( + TransferException $e, + public readonly array $failedPages, + public readonly LazyCollection $items, + ) { parent::__construct($e->getMessage(), 0, $e); } } diff --git a/src/Exceptions/UnsupportedPaginationException.php b/src/Exceptions/UnsupportedPaginationException.php index 5cfb6ee..16756e4 100644 --- a/src/Exceptions/UnsupportedPaginationException.php +++ b/src/Exceptions/UnsupportedPaginationException.php @@ -4,7 +4,7 @@ namespace Cerbero\LazyJsonPages\Exceptions; -use Cerbero\LazyJsonPages\Dtos\Config; +use Cerbero\LazyJsonPages\Data\Config; /** * The exception to throw when a pagination is not supported. diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index da76e78..a8c23f7 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -5,7 +5,7 @@ namespace Cerbero\LazyJsonPages; use Cerbero\LazyJson\Pointers\DotsConverter; -use Cerbero\LazyJsonPages\Dtos\Config; +use Cerbero\LazyJsonPages\Data\Config; use Cerbero\LazyJsonPages\Paginations\AnyPagination; use Cerbero\LazyJsonPages\Services\ClientFactory; use Cerbero\LazyJsonPages\Sources\AnySource; @@ -29,7 +29,7 @@ final class LazyJsonPages /** * The raw configuration of the API pagination. * - * @var array + * @var array */ private array $config = []; @@ -62,7 +62,7 @@ public function __construct(private readonly mixed $source) */ public function pageName(string $name): self { - $this->config['pageName'] = $name; + $this->config[Config::OPTION_PAGE_NAME] = $name; return $this; } @@ -72,7 +72,7 @@ public function pageName(string $name): self */ public function pageInPath(string $pattern = '/(\d+)(?!.*\d)/'): self { - $this->config['pageInPath'] = $pattern; + $this->config[Config::OPTION_PAGE_IN_PATH] = $pattern; return $this; } @@ -82,7 +82,7 @@ public function pageInPath(string $pattern = '/(\d+)(?!.*\d)/'): self */ public function firstPage(int $page): self { - $this->config['firstPage'] = max(0, $page); + $this->config[Config::OPTION_FIRST_PAGE] = max(0, $page); return $this; } @@ -92,7 +92,7 @@ public function firstPage(int $page): self */ public function totalPages(string $key): self { - $this->config['totalPagesKey'] = $key; + $this->config[Config::OPTION_TOTAL_PAGES_KEY] = $key; return $this; } @@ -102,7 +102,7 @@ public function totalPages(string $key): self */ public function totalItems(string $key): self { - $this->config['totalItemsKey'] = $key; + $this->config[Config::OPTION_TOTAL_ITEMS_KEY] = $key; return $this; } @@ -112,7 +112,7 @@ public function totalItems(string $key): self */ public function cursor(string $key): self { - $this->config['cursorKey'] = $key; + $this->config[Config::OPTION_CURSOR_KEY] = $key; return $this; } @@ -122,7 +122,7 @@ public function cursor(string $key): self */ public function lastPage(string $key): self { - $this->config['lastPageKey'] = $key; + $this->config[Config::OPTION_LAST_PAGE_KEY] = $key; return $this; } @@ -132,7 +132,7 @@ public function lastPage(string $key): self */ public function offset(string $key = 'offset'): self { - $this->config['offsetKey'] = $key; + $this->config[Config::OPTION_OFFSET_KEY] = $key; return $this; } @@ -142,7 +142,7 @@ public function offset(string $key = 'offset'): self */ public function linkHeader(): self { - $this->config['hasLinkHeader'] = true; + $this->config[Config::OPTION_HAS_LINK_HEADER] = true; return $this; } @@ -152,7 +152,23 @@ public function linkHeader(): self */ public function pagination(string $class): self { - $this->config['pagination'] = $class; + $this->config[Config::OPTION_PAGINATION] = $class; + + return $this; + } + + /** + * Throttle the requests to respect rate limiting. + */ + public function throttle(int $requests, int $perSeconds = 0, int $perMinutes = 0, int $perHours = 0): self + { + $seconds = $perSeconds + $perMinutes * 60 + $perHours * 3600; + + if ($requests > 0 && $seconds > 0) { + $this->config[Config::OPTION_RATE_LIMITS] ??= $this->client->rateLimits; + + $this->client->throttle($requests, $seconds); + } return $this; } @@ -162,7 +178,9 @@ public function pagination(string $class): self */ public function async(int $requests): self { - $this->config['async'] = max(1, $requests); + $this->config[Config::OPTION_ASYNC] = max(1, $requests); + + $this->client->config(RequestOptions::STREAM, $this->config[Config::OPTION_ASYNC] === 1); return $this; } @@ -174,6 +192,8 @@ public function connectionTimeout(float|int $seconds): self { $this->client->config(RequestOptions::CONNECT_TIMEOUT, max(0, $seconds)); + $this->client->config(RequestOptions::READ_TIMEOUT, max(0, $seconds)); + return $this; } @@ -183,7 +203,6 @@ public function connectionTimeout(float|int $seconds): self public function requestTimeout(float|int $seconds): self { $this->client->config(RequestOptions::TIMEOUT, max(0, $seconds)); - $this->client->config(RequestOptions::READ_TIMEOUT, max(0, $seconds)); return $this; } @@ -193,7 +212,7 @@ public function requestTimeout(float|int $seconds): self */ public function attempts(int $times): self { - $this->config['attempts'] = max(1, $times); + $this->config[Config::OPTION_ATTEMPTS] = max(1, $times); return $this; } @@ -203,7 +222,7 @@ public function attempts(int $times): self */ public function backoff(Closure $callback): self { - $this->config['backoff'] = $callback; + $this->config[Config::OPTION_BACKOFF] = $callback; return $this; } @@ -254,18 +273,6 @@ public function onError(Closure $callback): self return $this; } - /** - * Throttle the requests to respect rate limiting. - */ - public function throttle(int $requests, int $perSeconds = 0, int $perMinutes = 0, int $perHours = 0): self - { - $seconds = max(0, $perSeconds + $perMinutes * 60 + $perHours * 3600); - - $this->client->throttle($requests, $seconds); - - return $this; - } - /** * Retrieve a lazy collection yielding the paginated items. * @@ -274,7 +281,7 @@ public function throttle(int $requests, int $perSeconds = 0, int $perMinutes = 0 */ public function collect(string $dot = '*'): LazyCollection { - $this->config['itemsPointer'] = DotsConverter::toPointer($dot); + $this->config[Config::OPTION_ITEMS_POINTER] = DotsConverter::toPointer($dot); return new LazyCollection(function() { $client = $this->client->make(); diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 3d8e490..81be2eb 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -6,7 +6,7 @@ use Cerbero\LazyJsonPages\Concerns\ParsesPages; use Cerbero\LazyJsonPages\Concerns\ResolvesPages; -use Cerbero\LazyJsonPages\Dtos\Config; +use Cerbero\LazyJsonPages\Data\Config; use Cerbero\LazyJsonPages\Services\Book; use Cerbero\LazyJsonPages\Sources\AnySource; use GuzzleHttp\Client; diff --git a/src/Services/Book.php b/src/Services/Book.php index a956443..2bda315 100644 --- a/src/Services/Book.php +++ b/src/Services/Book.php @@ -20,7 +20,7 @@ final class Book /** * The pages unable to be fetched. * - * @var array + * @var int[] */ private array $failedPages = []; @@ -37,7 +37,7 @@ public function addPage(int $page, ResponseInterface $response): self /** * Yield and forget each page. * - * @return Generator + * @return Generator */ public function pullPages(): Generator { @@ -73,4 +73,12 @@ public function pullFailedPages(): array return $failedPages; } + + /** + * Retrieve and unset the oldest failed page. + */ + public function pullFailedPage(): ?int + { + return array_shift($this->failedPages); + } } diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php index 2e6d08a..b24bb24 100644 --- a/src/Services/ClientFactory.php +++ b/src/Services/ClientFactory.php @@ -48,6 +48,11 @@ final class ClientFactory */ private readonly TapCallbacks $tapCallbacks; + /** + * The rate limits. + */ + public readonly RateLimits $rateLimits; + /** * The custom client configuration. * @@ -62,13 +67,6 @@ final class ClientFactory */ private array $middleware = []; - /** - * The requests throttling. - * - * @var array - */ - private array $throttling = []; - /** * Add a global middleware. */ @@ -106,10 +104,13 @@ public static function fake(array $responses, Closure $callback): array public function __construct() { $this->tapCallbacks = new TapCallbacks(); + $this->rateLimits = new RateLimits(); } /** * Add the given option to the Guzzle client configuration. + * + * @param RequestOptions::* $name */ public function config(string $name, mixed $value): self { @@ -132,19 +133,11 @@ public function middleware(string $name, callable $middleware): self * Add the given callback to handle the sending request. */ public function onRequest(Closure $callback): self - { - $this->tapCallbacks->onRequest($callback); - - return $this->tap(); - } - - /** - * Add the middleware to handle a request before and after it is sent. - */ - private function tap(): self { $this->middleware['lazy_json_pages_tap'] ??= new Tap($this->tapCallbacks); + $this->tapCallbacks->onRequest($callback); + return $this; } @@ -153,9 +146,11 @@ private function tap(): self */ public function onResponse(Closure $callback): self { + $this->middleware['lazy_json_pages_tap'] ??= new Tap($this->tapCallbacks); + $this->tapCallbacks->onResponse($callback); - return $this->tap(); + return $this; } /** @@ -163,19 +158,21 @@ public function onResponse(Closure $callback): self */ public function onError(Closure $callback): self { + $this->middleware['lazy_json_pages_tap'] ??= new Tap($this->tapCallbacks); + $this->tapCallbacks->onError($callback); - return $this->tap(); + return $this; } /** * Throttle the requests to respect rate limiting. */ - public function throttle(int $requests, int $seconds): self + public function throttle(int $requests, int $perSeconds): self { - $this->throttling[$seconds] = $requests; + $this->rateLimits->add($requests, $perSeconds); - // $this->middleware['lazy_json_pages_throttle'] ??= Tap::once(); + $this->middleware['lazy_json_pages_throttle'] ??= Tap::once(fn() => $this->rateLimits->hit()); return $this; } diff --git a/src/Services/RateLimits.php b/src/Services/RateLimits.php new file mode 100644 index 0000000..3baa844 --- /dev/null +++ b/src/Services/RateLimits.php @@ -0,0 +1,74 @@ +rateLimits[] = new RateLimit($requests, $perSeconds); + + return $this; + } + + /** + * Hit the rate limits with a request. + */ + public function hit(): self + { + foreach ($this->rateLimits as $rateLimit) { + $rateLimit->hit(); + } + + return $this; + } + + /** + * Retrieve the number of requests allowed before the next rate limit. + */ + public function threshold(): int + { + $threshold = INF; + + foreach ($this->rateLimits as $rateLimit) { + $threshold = min($rateLimit->threshold(), $threshold); + } + + return (int) $threshold; + } + + /** + * Retrieve the timestamp after which it is possible to send new requests. + */ + public function resetAt(): float + { + $timestamp = 0.0; + + foreach ($this->rateLimits as $rateLimit) { + if ($rateLimit->wasReached()) { + $timestamp = max($rateLimit->resetsAt(), $timestamp); + + $rateLimit->reset(); + } + } + + return $timestamp; + } +} From d40a669a235f8915aaa87a81fb52db90d34c3be7 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 6 Sep 2024 13:43:23 -0300 Subject: [PATCH 097/109] Update readme --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 108ac98..8eaead6 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ When calling `collect()`, we indicate that the pagination structure is defined a ### ๐Ÿ’ง Sources -A source is any mean that can point to a paginated JSON API. A number of sources is supported by default: +A source is any means that can point to a paginated JSON API. A number of sources is supported by default: - **endpoint URIs**, e.g. `https://example.com/api/v1/users` or any instance of `Psr\Http\Message\UriInterface` - **PSR-7 requests**, i.e. any instance of `Psr\Http\Message\RequestInterface` @@ -307,7 +307,7 @@ If we need a middleware to be added every time we invoke Lazy JSON Pages, we can LazyJsonPages::globalMiddleware('fire_events', $fireEvents); ``` -Sometimes writing Guzzle middleware might be cumbersome, alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request, receiving a response or dealing with a transaction error: +Sometimes writing Guzzle middleware might be cumbersome. Alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request, receiving a response or dealing with a transaction error: ```php use Psr\Http\Message\RequestInterface; @@ -319,6 +319,15 @@ LazyJsonPages::from($source) ->onError(fn(Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); ``` +Several APIs set rate limits to limitate the number of allowed requests for a period of time. We can instruct Lazy JSON Pages to respect such limits by throttling our requests: + +```php +// we send a maximum of 3 requests per second, 60 per minute and 3,000 per hour +LazyJsonPages::from($source) + ->throttle(requests: 3, perSeconds: 1) + ->throttle(requests: 60, perMinutes: 1) + ->throttle(requests: 3000, perHours: 1); +``` ### ๐Ÿ’ข Errors handling From de070a9ce3a83d73681b7f11af5649119c021a3e Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 6 Sep 2024 14:12:34 -0300 Subject: [PATCH 098/109] Update instantiation vectors --- README.md | 19 +++++++++++++------ bootstrap.php | 8 ++++++++ composer.json | 1 + helpers.php | 11 ++--------- 4 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 bootstrap.php diff --git a/README.md b/README.md index 8eaead6..7133291 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,12 @@ [![Total Downloads][ico-downloads]][link-downloads] ```php -$lazyCollection = LazyJsonPages::from($source) +use Illuminate\Support\LazyCollection; + +LazyCollection::fromJsonPages($source) ->totalPages('pagination.total_pages') ->async(requests: 5) + ->throttle(requests: 60, perMinute: 1) ->collect('data.*'); ``` @@ -51,21 +54,25 @@ composer require cerbero/lazy-json-pages ### ๐Ÿ‘ฃ Basics -Depending on our coding style, we can initialize Lazy JSON Pages in 3 different ways: +Depending on our coding style, we can initialize Lazy JSON Pages in 4 different ways: ```php use Cerbero\LazyJsonPages\LazyJsonPages; +use Illuminate\Support\LazyCollection; use function Cerbero\LazyJsonPages\lazyJsonPages; +// lazy collection macro +LazyCollection::fromJsonPages($source); + // classic instantiation -$lazyJsonPages = new LazyJsonPages($source); +new LazyJsonPages($source); -// static method (easier methods chaining) -$lazyJsonPages = LazyJsonPages::from($source); +// static method +LazyJsonPages::from($source); // namespaced helper -$lazyJsonPages = lazyJsonPages($source); +lazyJsonPages($source); ``` The variable `$source` in our examples represents any [source](#-sources) that points to a paginated JSON API. Once we define the source, we can then chain methods to define how the API is paginated: diff --git a/bootstrap.php b/bootstrap.php new file mode 100644 index 0000000..5e9e82e --- /dev/null +++ b/bootstrap.php @@ -0,0 +1,8 @@ + Date: Fri, 6 Sep 2024 21:07:59 -0300 Subject: [PATCH 099/109] Fix style --- bootstrap.php | 2 +- src/Concerns/SendsAsyncRequests.php | 4 ++-- src/Concerns/YieldsItemsByCursor.php | 2 +- src/Concerns/YieldsItemsByLength.php | 2 +- src/Data/RateLimit.php | 3 +-- src/LazyJsonPages.php | 2 +- src/Middleware/Tap.php | 4 +--- src/Paginations/CursorAwarePagination.php | 2 +- src/Paginations/LastPageAwarePagination.php | 2 +- src/Paginations/LinkHeaderAwarePagination.php | 4 ++-- src/Paginations/TotalItemsAwarePagination.php | 2 +- src/Services/ClientFactory.php | 1 - src/Sources/LaravelClientRequest.php | 2 +- src/Sources/LaravelClientResponse.php | 2 +- 14 files changed, 15 insertions(+), 19 deletions(-) diff --git a/bootstrap.php b/bootstrap.php index 5e9e82e..593071d 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -3,6 +3,6 @@ use Cerbero\LazyJsonPages\LazyJsonPages; use Illuminate\Support\LazyCollection; -(static function() { +(static function () { LazyCollection::macro('fromJsonPages', [LazyJsonPages::class, 'from']); })(); diff --git a/src/Concerns/SendsAsyncRequests.php b/src/Concerns/SendsAsyncRequests.php index 273ff7a..25c8a5c 100644 --- a/src/Concerns/SendsAsyncRequests.php +++ b/src/Concerns/SendsAsyncRequests.php @@ -32,7 +32,7 @@ protected function fetchPagesAsynchronously(int $totalPages): Generator $fromPage = $this->config->firstPage + 1; $toPage = $this->config->firstPage == 0 ? $totalPages - 1 : $totalPages; - yield from $this->retry(function() use ($request, &$fromPage, $toPage) { + yield from $this->retry(function () use ($request, &$fromPage, $toPage) { foreach ($this->chunkRequestsBetweenPages($request, $fromPage, $toPage) as $requests) { yield from $this->pool($requests); } @@ -83,7 +83,7 @@ protected function pool(Generator $requests): Generator $config = [ 'concurrency' => $this->config->async, 'fulfilled' => fn(ResponseInterface $response, int $page) => $this->book->addPage($page, $response), - 'rejected' => function(Throwable $e, int $page) use (&$exception) { + 'rejected' => function (Throwable $e, int $page) use (&$exception) { $this->book->addFailedPage($page); $exception = $e; }, diff --git a/src/Concerns/YieldsItemsByCursor.php b/src/Concerns/YieldsItemsByCursor.php index 2ae9a8f..df3a426 100644 --- a/src/Concerns/YieldsItemsByCursor.php +++ b/src/Concerns/YieldsItemsByCursor.php @@ -28,7 +28,7 @@ protected function yieldItemsByCursor(Closure $callback): Generator yield from $generator = $callback($this->source->pullResponse()); - yield from $this->retry(function() use ($callback, $request, $generator) { + yield from $this->retry(function () use ($callback, $request, $generator) { while ($cursor = $this->toPage($generator->getReturn(), onlyNumerics: false)) { $this->respectRateLimits(); diff --git a/src/Concerns/YieldsItemsByLength.php b/src/Concerns/YieldsItemsByLength.php index 188e995..c9351f6 100644 --- a/src/Concerns/YieldsItemsByLength.php +++ b/src/Concerns/YieldsItemsByLength.php @@ -24,7 +24,7 @@ trait YieldsItemsByLength */ protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): Generator { - yield from $this->yieldItemsUntilPage(function(ResponseInterface $response) use ($key, $callback) { + yield from $this->yieldItemsUntilPage(function (ResponseInterface $response) use ($key, $callback) { yield from $generator = $this->yieldItemsAndGetKey($response, $key); if (!is_int($page = $this->toPage($generator->getReturn()))) { diff --git a/src/Data/RateLimit.php b/src/Data/RateLimit.php index 99ecb29..68f24e9 100644 --- a/src/Data/RateLimit.php +++ b/src/Data/RateLimit.php @@ -25,8 +25,7 @@ final class RateLimit public function __construct( public readonly int $requests, public readonly int $perSeconds, - ) { - } + ) {} /** * Update the requests sent before this rate limit resets. diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index a8c23f7..80f2bb4 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -283,7 +283,7 @@ public function collect(string $dot = '*'): LazyCollection { $this->config[Config::OPTION_ITEMS_POINTER] = DotsConverter::toPointer($dot); - return new LazyCollection(function() { + return new LazyCollection(function () { $client = $this->client->make(); $config = new Config(...$this->config); $source = (new AnySource($this->source))->setClient($client); diff --git a/src/Middleware/Tap.php b/src/Middleware/Tap.php index b5b5480..3b00d18 100644 --- a/src/Middleware/Tap.php +++ b/src/Middleware/Tap.php @@ -45,9 +45,7 @@ public static function once(?Closure $onRequest = null, ?Closure $onResponse = n /** * Instantiate the class. */ - public function __construct(private readonly TapCallbacks $callbacks) - { - } + public function __construct(private readonly TapCallbacks $callbacks) {} /** * Handle an HTTP request before and after it is sent. diff --git a/src/Paginations/CursorAwarePagination.php b/src/Paginations/CursorAwarePagination.php index 2448b14..076bf9d 100644 --- a/src/Paginations/CursorAwarePagination.php +++ b/src/Paginations/CursorAwarePagination.php @@ -33,7 +33,7 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsByCursor(function(ResponseInterface $response) { + yield from $this->yieldItemsByCursor(function (ResponseInterface $response) { yield from $generator = $this->yieldItemsAndGetKey($response, $this->config->cursorKey); return $generator->getReturn(); diff --git a/src/Paginations/LastPageAwarePagination.php b/src/Paginations/LastPageAwarePagination.php index 94a6224..21bee16 100644 --- a/src/Paginations/LastPageAwarePagination.php +++ b/src/Paginations/LastPageAwarePagination.php @@ -29,7 +29,7 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function(int $page) { + yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function (int $page) { return $this->config->firstPage === 0 ? $page + 1 : $page; }); } diff --git a/src/Paginations/LinkHeaderAwarePagination.php b/src/Paginations/LinkHeaderAwarePagination.php index 3e0a4c0..57f2d40 100644 --- a/src/Paginations/LinkHeaderAwarePagination.php +++ b/src/Paginations/LinkHeaderAwarePagination.php @@ -81,7 +81,7 @@ protected function parseLinkHeader(string $linkHeader, ?string $relation = null) */ protected function yieldItemsByLastPage(int $lastPage): Generator { - yield from $this->yieldItemsUntilPage(function(ResponseInterface $response) use ($lastPage) { + yield from $this->yieldItemsUntilPage(function (ResponseInterface $response) use ($lastPage) { yield from $this->yieldItemsFrom($response); return $this->config->firstPage === 0 ? $lastPage + 1 : $lastPage; @@ -95,7 +95,7 @@ protected function yieldItemsByLastPage(int $lastPage): Generator */ protected function yieldItemsByNextLink(): Generator { - yield from $this->yieldItemsByCursor(function(ResponseInterface $response) { + yield from $this->yieldItemsByCursor(function (ResponseInterface $response) { yield from $this->yieldItemsFrom($response); return $this->parseLinkHeader($response->getHeaderLine('link'), 'next'); diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 2241347..9c9325c 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -31,7 +31,7 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsUntilKey($this->config->totalItemsKey, function(int $totalItems) { + yield from $this->yieldItemsUntilKey($this->config->totalItemsKey, function (int $totalItems) { return $this->itemsPerPage > 0 ? (int) ceil($totalItems / $this->itemsPerPage) : 0; }); } diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php index b24bb24..142a9ff 100644 --- a/src/Services/ClientFactory.php +++ b/src/Services/ClientFactory.php @@ -5,7 +5,6 @@ namespace Cerbero\LazyJsonPages\Services; use Cerbero\LazyJsonPages\Middleware\Tap; -use Cerbero\LazyJsonPages\Services\TapCallbacks; use Closure; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; diff --git a/src/Sources/LaravelClientRequest.php b/src/Sources/LaravelClientRequest.php index ed1eb85..ab37c61 100644 --- a/src/Sources/LaravelClientRequest.php +++ b/src/Sources/LaravelClientRequest.php @@ -4,9 +4,9 @@ namespace Cerbero\LazyJsonPages\Sources; +use Illuminate\Http\Client\Request; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; -use Illuminate\Http\Client\Request; /** * The Laravel HTTP client source. diff --git a/src/Sources/LaravelClientResponse.php b/src/Sources/LaravelClientResponse.php index 2490858..2e4f9c9 100644 --- a/src/Sources/LaravelClientResponse.php +++ b/src/Sources/LaravelClientResponse.php @@ -5,9 +5,9 @@ namespace Cerbero\LazyJsonPages\Sources; use Cerbero\LazyJsonPages\Exceptions\RequestNotSentException; +use Illuminate\Http\Client\Response; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; -use Illuminate\Http\Client\Response; /** * The Laravel HTTP client source. From d4c730e863a5a8dc316d2422fa409438bbec1992 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Fri, 6 Sep 2024 23:28:07 -0300 Subject: [PATCH 100/109] Improve static analysis --- src/Concerns/ParsesPages.php | 4 +++- src/Concerns/ResolvesPages.php | 7 +++++-- src/Concerns/RetriesHttpRequests.php | 8 +++++--- src/Concerns/YieldsItemsByLength.php | 9 ++++++--- src/Data/RateLimit.php | 2 +- src/Exceptions/OutOfAttemptsException.php | 3 +-- src/LazyJsonPages.php | 4 ++-- src/Middleware/Tap.php | 2 +- src/Paginations/CursorAwarePagination.php | 1 + src/Paginations/LastPageAwarePagination.php | 1 + src/Paginations/LinkHeaderAwarePagination.php | 6 ++---- src/Paginations/TotalItemsAwarePagination.php | 1 + src/Paginations/TotalPagesAwarePagination.php | 2 +- src/Providers/LazyJsonPagesServiceProvider.php | 2 ++ src/Services/RateLimits.php | 2 +- src/Sources/AnySource.php | 2 +- src/Sources/Endpoint.php | 4 ++-- src/Sources/Source.php | 2 +- src/Sources/SymfonyRequest.php | 2 +- 19 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/Concerns/ParsesPages.php b/src/Concerns/ParsesPages.php index 6cdb426..f267906 100644 --- a/src/Concerns/ParsesPages.php +++ b/src/Concerns/ParsesPages.php @@ -17,7 +17,7 @@ trait ParsesPages /** * The number of items per page. */ - protected readonly int $itemsPerPage; + protected int $itemsPerPage; /** * Yield paginated items and the given key from the provided response. @@ -35,6 +35,7 @@ protected function yieldItemsAndGetKey(ResponseInterface $response, string $key) foreach (JsonParser::parse($response)->pointers($pointers) as $item) { if (is_object($item)) { + /** @var object{value: mixed} $item */ $value = $item->value; } else { yield $item; @@ -54,6 +55,7 @@ protected function yieldItemsAndGetKey(ResponseInterface $response, string $key) */ protected function yieldItemsFrom(mixed $source): Generator { + /** @phpstan-ignore-next-line */ yield from JsonParser::parse($source)->pointer($this->config->itemsPointer); } } diff --git a/src/Concerns/ResolvesPages.php b/src/Concerns/ResolvesPages.php index e0d8dda..50ca8df 100644 --- a/src/Concerns/ResolvesPages.php +++ b/src/Concerns/ResolvesPages.php @@ -27,13 +27,14 @@ protected function toPage(mixed $value, bool $onlyNumerics = true): string|int|n is_numeric($value) => (int) $value, !is_string($value) || $value === '' => null, !$this->isEndpoint($value) => $onlyNumerics ? null : $value, - default => $this->pageFromParsedUri(parse_url($value), $onlyNumerics), + default => $this->pageFromParsedUri(parse_url($value), $onlyNumerics), /** @phpstan-ignore-line */ }; } /** * Retrieve the page from the given parsed URI. * + * @param array{path?: string, query?: string} $parsedUri * @return ($onlyNumerics is true ? int|null : string|int|null) */ protected function pageFromParsedUri(array $parsedUri, bool $onlyNumerics = true): string|int|null @@ -55,7 +56,9 @@ protected function pageFromParsedUri(array $parsedUri, bool $onlyNumerics = true protected function uriForPage(UriInterface $uri, string $page): UriInterface { if ($key = $this->config->offsetKey) { - return Uri::withQueryValue($uri, $key, strval(($page - $this->config->firstPage) * $this->itemsPerPage)); + $value = (intval($page) - $this->config->firstPage) * $this->itemsPerPage; + + return Uri::withQueryValue($uri, $key, strval($value)); } if (!$pattern = $this->config->pageInPath) { diff --git a/src/Concerns/RetriesHttpRequests.php b/src/Concerns/RetriesHttpRequests.php index d98b9a4..add5432 100644 --- a/src/Concerns/RetriesHttpRequests.php +++ b/src/Concerns/RetriesHttpRequests.php @@ -5,6 +5,7 @@ namespace Cerbero\LazyJsonPages\Concerns; use Cerbero\LazyJsonPages\Exceptions\OutOfAttemptsException; +use Closure; use Generator; use GuzzleHttp\Exception\TransferException; use Illuminate\Support\LazyCollection; @@ -20,10 +21,11 @@ trait RetriesHttpRequests /** * Retry to yield HTTP responses from the given callback. * - * @param callable $callback - * @return Generator + * @template TGen of Generator + * @param Closure(): TGen $callback + * @return TGen */ - protected function retry(callable $callback): Generator + protected function retry(Closure $callback): Generator { $attempt = 0; $remainingAttempts = $this->config->attempts; diff --git a/src/Concerns/YieldsItemsByLength.php b/src/Concerns/YieldsItemsByLength.php index c9351f6..9e6b386 100644 --- a/src/Concerns/YieldsItemsByLength.php +++ b/src/Concerns/YieldsItemsByLength.php @@ -20,7 +20,7 @@ trait YieldsItemsByLength * Yield paginated items until the page resolved from the given key is reached. * * @param ?Closure(int): int $callback - * @return Generator + * @return Generator */ protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): Generator { @@ -38,14 +38,17 @@ protected function yieldItemsUntilKey(string $key, ?Closure $callback = null): G /** * Yield paginated items until the resolved page is reached. * - * @param Closure(ResponseInterface): Generator $callback + * @param Closure(ResponseInterface): Generator $callback * @return Generator */ protected function yieldItemsUntilPage(Closure $callback): Generator { yield from $generator = $callback($this->source->pullResponse()); - foreach ($this->fetchPagesAsynchronously($generator->getReturn()) as $response) { + /** @var int */ + $totalPages = $generator->getReturn(); + + foreach ($this->fetchPagesAsynchronously($totalPages) as $response) { yield from $this->yieldItemsFrom($response); } } diff --git a/src/Data/RateLimit.php b/src/Data/RateLimit.php index 68f24e9..6489ed2 100644 --- a/src/Data/RateLimit.php +++ b/src/Data/RateLimit.php @@ -57,7 +57,7 @@ public function wasReached(): bool /** * Retrieve the timestamp when this rate limit resets. */ - public function resetsAt(): float + public function resetsAt(): ?float { return $this->resetsAt; } diff --git a/src/Exceptions/OutOfAttemptsException.php b/src/Exceptions/OutOfAttemptsException.php index b6b0aa6..ae64b9a 100644 --- a/src/Exceptions/OutOfAttemptsException.php +++ b/src/Exceptions/OutOfAttemptsException.php @@ -2,7 +2,6 @@ namespace Cerbero\LazyJsonPages\Exceptions; -use Closure; use GuzzleHttp\Exception\TransferException; use Illuminate\Support\LazyCollection; @@ -15,7 +14,7 @@ class OutOfAttemptsException extends LazyJsonPagesException * Instantiate the class. * * @param array $failedPages - * @param (Closure(): Generator) $items + * @param LazyCollection $items */ public function __construct( TransferException $e, diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index 80f2bb4..a81c1ae 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -277,7 +277,7 @@ public function onError(Closure $callback): self * Retrieve a lazy collection yielding the paginated items. * * @return LazyCollection - * @throws UnsupportedPaginationException + * @throws \Cerbero\LazyJsonPages\Exceptions\UnsupportedPaginationException */ public function collect(string $dot = '*'): LazyCollection { @@ -285,7 +285,7 @@ public function collect(string $dot = '*'): LazyCollection return new LazyCollection(function () { $client = $this->client->make(); - $config = new Config(...$this->config); + $config = new Config(...$this->config); /** @phpstan-ignore-line */ $source = (new AnySource($this->source))->setClient($client); yield from new AnyPagination($source, $client, $config); diff --git a/src/Middleware/Tap.php b/src/Middleware/Tap.php index 3b00d18..e29bdf7 100644 --- a/src/Middleware/Tap.php +++ b/src/Middleware/Tap.php @@ -50,7 +50,7 @@ public function __construct(private readonly TapCallbacks $callbacks) {} /** * Handle an HTTP request before and after it is sent. * - * @param callable(RequestInterface, array): PromiseInterface $handler + * @param callable(RequestInterface, array): PromiseInterface $handler */ public function __invoke(callable $handler): Closure { diff --git a/src/Paginations/CursorAwarePagination.php b/src/Paginations/CursorAwarePagination.php index 076bf9d..713339d 100644 --- a/src/Paginations/CursorAwarePagination.php +++ b/src/Paginations/CursorAwarePagination.php @@ -34,6 +34,7 @@ public function matches(): bool public function getIterator(): Traversable { yield from $this->yieldItemsByCursor(function (ResponseInterface $response) { + /** @phpstan-ignore-next-line */ yield from $generator = $this->yieldItemsAndGetKey($response, $this->config->cursorKey); return $generator->getReturn(); diff --git a/src/Paginations/LastPageAwarePagination.php b/src/Paginations/LastPageAwarePagination.php index 21bee16..f9954a4 100644 --- a/src/Paginations/LastPageAwarePagination.php +++ b/src/Paginations/LastPageAwarePagination.php @@ -29,6 +29,7 @@ public function matches(): bool */ public function getIterator(): Traversable { + /** @phpstan-ignore-next-line */ yield from $this->yieldItemsUntilKey($this->config->lastPageKey, function (int $page) { return $this->config->firstPage === 0 ? $page + 1 : $page; }); diff --git a/src/Paginations/LinkHeaderAwarePagination.php b/src/Paginations/LinkHeaderAwarePagination.php index 57f2d40..6d42e25 100644 --- a/src/Paginations/LinkHeaderAwarePagination.php +++ b/src/Paginations/LinkHeaderAwarePagination.php @@ -6,7 +6,6 @@ use Cerbero\LazyJsonPages\Concerns\YieldsItemsByCursor; use Cerbero\LazyJsonPages\Concerns\YieldsItemsByLength; -use Cerbero\LazyJsonPages\Exceptions\InvalidLinkHeaderException; use Generator; use Psr\Http\Message\ResponseInterface; use Traversable; @@ -39,10 +38,10 @@ public function matches(): bool * Yield the paginated items. * * @return Traversable - * @throws InvalidLinkHeaderException */ public function getIterator(): Traversable { + /** @var array{last?: int, next?: string|int} */ $links = $this->parseLinkHeader($this->source->response()->getHeaderLine('link')); yield from match (true) { @@ -55,11 +54,10 @@ public function getIterator(): Traversable /** * Retrieve the parsed Link header. * - * @template TParsed of array{last?: int, next?: string|int} * @template TRelation of string|null * * @param TRelation $relation - * @return (TRelation is null ? TParsed : string|int|null) + * @return (TRelation is null ? array : string|int|null) */ protected function parseLinkHeader(string $linkHeader, ?string $relation = null): array|string|int|null { diff --git a/src/Paginations/TotalItemsAwarePagination.php b/src/Paginations/TotalItemsAwarePagination.php index 9c9325c..dacd1b7 100644 --- a/src/Paginations/TotalItemsAwarePagination.php +++ b/src/Paginations/TotalItemsAwarePagination.php @@ -31,6 +31,7 @@ public function matches(): bool */ public function getIterator(): Traversable { + /** @phpstan-ignore-next-line */ yield from $this->yieldItemsUntilKey($this->config->totalItemsKey, function (int $totalItems) { return $this->itemsPerPage > 0 ? (int) ceil($totalItems / $this->itemsPerPage) : 0; }); diff --git a/src/Paginations/TotalPagesAwarePagination.php b/src/Paginations/TotalPagesAwarePagination.php index 824aae9..4673238 100644 --- a/src/Paginations/TotalPagesAwarePagination.php +++ b/src/Paginations/TotalPagesAwarePagination.php @@ -29,6 +29,6 @@ public function matches(): bool */ public function getIterator(): Traversable { - yield from $this->yieldItemsUntilKey($this->config->totalPagesKey); + yield from $this->yieldItemsUntilKey($this->config->totalPagesKey); /** @phpstan-ignore-line */ } } diff --git a/src/Providers/LazyJsonPagesServiceProvider.php b/src/Providers/LazyJsonPagesServiceProvider.php index 01a3c46..e46d46f 100644 --- a/src/Providers/LazyJsonPagesServiceProvider.php +++ b/src/Providers/LazyJsonPagesServiceProvider.php @@ -18,6 +18,8 @@ /** * The service provider to integrate with Laravel. + * + * @property-read array{events: \Illuminate\Contracts\Events\Dispatcher} $app */ final class LazyJsonPagesServiceProvider extends ServiceProvider { diff --git a/src/Services/RateLimits.php b/src/Services/RateLimits.php index 3baa844..1674401 100644 --- a/src/Services/RateLimits.php +++ b/src/Services/RateLimits.php @@ -63,7 +63,7 @@ public function resetAt(): float foreach ($this->rateLimits as $rateLimit) { if ($rateLimit->wasReached()) { - $timestamp = max($rateLimit->resetsAt(), $timestamp); + $timestamp = max($rateLimit->resetsAt() ?? 0.0, $timestamp); $rateLimit->reset(); } diff --git a/src/Sources/AnySource.php b/src/Sources/AnySource.php index 9cf4614..bd69646 100644 --- a/src/Sources/AnySource.php +++ b/src/Sources/AnySource.php @@ -28,7 +28,7 @@ class AnySource extends Source /** * The matching source. */ - protected readonly Source $matchingSource; + protected Source $matchingSource; /** * The cached HTTP response. diff --git a/src/Sources/Endpoint.php b/src/Sources/Endpoint.php index 74ecb31..e864fab 100644 --- a/src/Sources/Endpoint.php +++ b/src/Sources/Endpoint.php @@ -24,8 +24,8 @@ class Endpoint extends Source */ public function matches(): bool { - return $this->source instanceof UriInterface - || (is_string($this->source) && $this->isEndpoint($this->source)); + return (is_string($this->source) && $this->isEndpoint($this->source)) + || $this->source instanceof UriInterface; } /** diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 4f4f677..21021ff 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -16,7 +16,7 @@ abstract class Source /** * The HTTP client. */ - protected readonly Client $client; + protected Client $client; /** * Retrieve the HTTP request. diff --git a/src/Sources/SymfonyRequest.php b/src/Sources/SymfonyRequest.php index fba5ebc..3ca2211 100644 --- a/src/Sources/SymfonyRequest.php +++ b/src/Sources/SymfonyRequest.php @@ -32,7 +32,7 @@ public function request(): RequestInterface return new Psr7Request( $this->source->getMethod(), $this->source->getUri(), - $this->source->headers->all(), + $this->source->headers->all(), /** @phpstan-ignore-line */ $this->source->getContent() ?: null, ); } From 9dde45fe3eea00755fe3864ad23904c0c5938915 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 9 Sep 2024 11:37:27 -0300 Subject: [PATCH 101/109] Update dependencies --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0845105..316c572 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: php: [8.1, 8.2, 8.3] - dependency-version: [prefer-lowest, prefer-stable] + dependency-version: [prefer-stable] os: [ubuntu-latest] name: PHP ${{ matrix.php }} - ${{ matrix.dependency-version }} - ${{ matrix.os }} From 55c36878d3ce47191dc519703f7d038fa85785fe Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 9 Sep 2024 13:41:00 -0300 Subject: [PATCH 102/109] Move method --- src/LazyJsonPages.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/LazyJsonPages.php b/src/LazyJsonPages.php index a81c1ae..5857214 100644 --- a/src/LazyJsonPages.php +++ b/src/LazyJsonPages.php @@ -108,21 +108,21 @@ public function totalItems(string $key): self } /** - * Set the cursor or next page. + * Set the number of the last page. */ - public function cursor(string $key): self + public function lastPage(string $key): self { - $this->config[Config::OPTION_CURSOR_KEY] = $key; + $this->config[Config::OPTION_LAST_PAGE_KEY] = $key; return $this; } /** - * Set the number of the last page. + * Set the cursor or next page. */ - public function lastPage(string $key): self + public function cursor(string $key): self { - $this->config[Config::OPTION_LAST_PAGE_KEY] = $key; + $this->config[Config::OPTION_CURSOR_KEY] = $key; return $this; } From cc905cdbbe1d0740dbd9fc6a80a4c389bfda0ac3 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 9 Sep 2024 13:41:32 -0300 Subject: [PATCH 103/109] Update readme --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7133291..50003d6 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ use Illuminate\Support\LazyCollection; LazyCollection::fromJsonPages($source) ->totalPages('pagination.total_pages') ->async(requests: 5) - ->throttle(requests: 60, perMinute: 1) + ->throttle(requests: 100, perMinutes: 1) ->collect('data.*'); ``` -Framework-agnostic package to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. +Framework-agnostic API scraper to load items from any paginated JSON API into a [Laravel lazy collection](https://laravel.com/docs/collections#lazy-collections) via async HTTP requests. > [!TIP] > Need to read large JSON with no pagination in a memory-efficient way? @@ -102,6 +102,9 @@ A source is any means that can point to a paginated JSON API. A number of source Here are some examples of sources: ```php +// a simple URI string +$source = 'https://example.com/api'; + // any PSR-7 compatible request is supported, including Guzzle requests $source = new GuzzleHttp\Psr7\Request('GET', 'https://example.com/api'); @@ -111,6 +114,8 @@ $source = Http::withToken($bearer)->get('https://example.com/api'); If none of the above sources satifies our use case, we can implement our own source. +
Click here to see how to implement a custom source. + To implement a custom source, we need to extend `Source` and implement 2 methods: ```php @@ -136,7 +141,7 @@ The parent class `Source` gives us access to 2 properties: - `$source`: the custom source for our use case - `$client`: the Guzzle HTTP client -The methods to implement respectively turn our custom source into a PSR-7 request and a PSR-7 response. Please refer to the [already existing sources](https://github.com/cerbero90/json-parser/tree/master/src/Sources) to see some implementations. +The methods to implement turn our custom source into a PSR-7 request and a PSR-7 response. Please refer to the [already existing sources](https://github.com/cerbero90/json-parser/tree/master/src/Sources) to see some implementations. Once the custom source is implemented, we can instruct Lazy JSON Pages to use it: @@ -145,6 +150,7 @@ LazyJsonPages::from(new CustomSource($source)); ``` If you find yourself implementing the same custom source in different projects, feel free to send a PR and we will consider to support your custom source by default. Thank you in advance for any contribution! +
### ๐Ÿ›๏ธ Pagination structure @@ -248,7 +254,7 @@ Some paginated API responses include a header called `Link`. An example is [GitH ; rel="last" ``` -To lazy-load the items of a Link header pagination, we can chain the method `linkHeader()`: +To lazy-load items from a Link header pagination, we can chain the method `linkHeader()`: ```php LazyJsonPages::from($source)->linkHeader(); @@ -259,6 +265,8 @@ LazyJsonPages::from($source)->linkHeader(); Lazy JSON Pages provides several methods to extract items from the most popular pagination mechanisms. However if we need a custom solution, we can implement our own pagination. +
Click here to see how to implement a custom source. + To implement a custom pagination, we need to extend `Pagination` and implement 1 method: ```php @@ -288,6 +296,7 @@ LazyJsonPages::from($source)->pagination(CustomPagination::class); ``` If you find yourself implementing the same custom pagination in different projects, feel free to send a PR and we will consider to support your custom pagination by default. Thank you in advance for any contribution! +
### ๐Ÿš€ Requests optimization @@ -300,6 +309,19 @@ By default HTTP requests are sent synchronously. If we want to send more than on LazyJsonPages::from($source)->async(requests: 5); ``` +> [!NOTE] +> Please note that asynchronous requests improve speed at the expense of memory, as more responses are going to be loaded at once. + +Several APIs set rate limits to reduce the number of allowed requests for a period of time. We can instruct Lazy JSON Pages to respect such limits by throttling our requests: + +```php +// we send a maximum of 3 requests per second, 60 per minute and 3,000 per hour +LazyJsonPages::from($source) + ->throttle(requests: 3, perSeconds: 1) + ->throttle(requests: 60, perMinutes: 1) + ->throttle(requests: 3000, perHours: 1); +``` + Internally, Lazy JSON Pages uses [Guzzle](https://docs.guzzlephp.org) as its HTTP client. We can customize the client behavior by adding as many [middleware](https://docs.guzzlephp.org/en/stable/handlers-and-middleware.html#middleware) as we need: ```php @@ -314,32 +336,69 @@ If we need a middleware to be added every time we invoke Lazy JSON Pages, we can LazyJsonPages::globalMiddleware('fire_events', $fireEvents); ``` -Sometimes writing Guzzle middleware might be cumbersome. Alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request, receiving a response or dealing with a transaction error: +Sometimes writing [Guzzle middleware](https://docs.guzzlephp.org/en/stable/handlers-and-middleware.html#middleware) might be cumbersome. Alternatively Lazy JSON Pages provides convenient methods to fire callbacks when sending a request or receiving a response: ```php use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; LazyJsonPages::from($source) - ->onRequest(fn(RequestInterface $request) => ...) - ->onResponse(fn(ResponseInterface $response, RequestInterface $request) => ...) - ->onError(fn(Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); + ->onRequest(fn (RequestInterface $request) => ...) + ->onResponse(fn (ResponseInterface $response, RequestInterface $request) => ...); ``` -Several APIs set rate limits to limitate the number of allowed requests for a period of time. We can instruct Lazy JSON Pages to respect such limits by throttling our requests: +We can also tweak the number of allowed seconds before an API connection times out or the allowed duration of the entire HTTP request (by default they are both set to 5 seconds): ```php -// we send a maximum of 3 requests per second, 60 per minute and 3,000 per hour LazyJsonPages::from($source) - ->throttle(requests: 3, perSeconds: 1) - ->throttle(requests: 60, perMinutes: 1) - ->throttle(requests: 3000, perHours: 1); + ->connectionTimeout(7) + ->requestTimeout(10); +``` + +If the 3rd party API is faulty or error-prone, we can indicate how many times we want to retry failing HTTP requests and the backoff strategy to compute the milliseconds to wait before retrying (by default failing requests are repeated 3 times after an exponential backoff of 100, 400 and 900 milliseconds): + +```php +// repeat failing requests 5 times after a backoff of 1, 2, 3, 4 and 5 seconds +LazyJsonPages::from($source) + ->attempts(5) + ->backoff(fn (int $attempt) => $attempt * 1000); ``` ### ๐Ÿ’ข Errors handling -> [!WARNING] -> The documentation of this feature is a work in progress. +If something goes wrong during the scraping process, we can intercept the error and execute a custom logic to handle it: + +```php +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; + +LazyJsonPages::from($source) + ->onError(fn (Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); +``` + +Any exception thrown by this package extends the `LazyJsonPagesException` class. This makes it easy to handle all exceptions in a single catch block: + +```php +use Cerbero\LazyJsonPages\Exceptions\LazyJsonPagesException; + +try { + LazyJsonPages::from($source)->cursor('cursor')->collect()->each(fn () => ...); +} catch (LazyJsonPagesException $e) { + // handle any exception thrown by Lazy JSON Pages +} +``` + +For reference, here is a comprehensive table of all the exceptions thrown by this package: + +|`Cerbero\LazyJsonPages\Exceptions\`|thrown when| +|---|---| +|`InvalidKeyException`|a JSON key does not contain a valid value| +|`InvalidPageInPathException`|a page cannot be found in the URI path| +|`InvalidPaginationException`|a pagination implementation is not valid| +|`OutOfAttemptsException`|an HTTP request failed too many times| +|`RequestNotSentException`|a JSON source didn't send any HTTP request| +|`UnsupportedPaginationException`|a pagination is not supported| +|`UnsupportedSourceException`|a JSON source is not supported| ### ๐Ÿค Laravel integration From 38afb2fe0a4118710f0556b10d15c2f395c547ba Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Mon, 9 Sep 2024 13:54:50 -0300 Subject: [PATCH 104/109] Update readme --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 50003d6..5178936 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ use Illuminate\Support\LazyCollection; LazyCollection::fromJsonPages($source) ->totalPages('pagination.total_pages') - ->async(requests: 5) + ->async(requests: 3) ->throttle(requests: 100, perMinutes: 1) ->collect('data.*'); ``` @@ -54,7 +54,7 @@ composer require cerbero/lazy-json-pages ### ๐Ÿ‘ฃ Basics -Depending on our coding style, we can initialize Lazy JSON Pages in 4 different ways: +Depending on our coding style, we can instantiate Lazy JSON Pages in 4 different ways: ```php use Cerbero\LazyJsonPages\LazyJsonPages; @@ -103,13 +103,13 @@ Here are some examples of sources: ```php // a simple URI string -$source = 'https://example.com/api'; +$source = 'https://example.com/api/v1/users'; // any PSR-7 compatible request is supported, including Guzzle requests -$source = new GuzzleHttp\Psr7\Request('GET', 'https://example.com/api'); +$source = new GuzzleHttp\Psr7\Request('GET', 'https://example.com/api/v1/users'); // while being framework-agnostic, Lazy JSON Pages integrates well with Laravel -$source = Http::withToken($bearer)->get('https://example.com/api'); +$source = Http::withToken($bearer)->get('https://example.com/api/v1/users'); ``` If none of the above sources satifies our use case, we can implement our own source. @@ -265,7 +265,7 @@ LazyJsonPages::from($source)->linkHeader(); Lazy JSON Pages provides several methods to extract items from the most popular pagination mechanisms. However if we need a custom solution, we can implement our own pagination. -
Click here to see how to implement a custom source. +
Click here to see how to implement a custom pagination. To implement a custom pagination, we need to extend `Pagination` and implement 1 method: @@ -382,7 +382,7 @@ Any exception thrown by this package extends the `LazyJsonPagesException` class. use Cerbero\LazyJsonPages\Exceptions\LazyJsonPagesException; try { - LazyJsonPages::from($source)->cursor('cursor')->collect()->each(fn () => ...); + LazyJsonPages::from($source)->linkHeader()->collect()->each(...); } catch (LazyJsonPagesException $e) { // handle any exception thrown by Lazy JSON Pages } From e398e2fdd7164e9b1303498ba51359b3604773c8 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 10 Sep 2024 20:11:28 -0300 Subject: [PATCH 105/109] Update dependencies --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 99980aa..f9c62fc 100644 --- a/composer.json +++ b/composer.json @@ -22,11 +22,11 @@ "require": { "php": "^8.1", "cerbero/lazy-json": "^2.0", - "guzzlehttp/guzzle": "^7.2" + "guzzlehttp/guzzle": "^7.2", + "illuminate/support": ">=6.20" }, "require-dev": { "illuminate/http": ">=6.20", - "illuminate/support": ">=6.20", "mockery/mockery": "^1.3.4", "orchestra/testbench": ">=7.0", "pestphp/pest": "^2.0", From 577edd2b3f3aa76011b739bea6306d9abd174fce Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 10 Sep 2024 20:13:18 -0300 Subject: [PATCH 106/109] Improve code coverage --- src/Concerns/RespectsRateLimits.php | 6 +- src/Paginations/Pagination.php | 2 + src/Services/ClientFactory.php | 26 +++++++ src/Sources/Source.php | 2 + tests/Feature/PaginationTest.php | 41 ++++++++++ tests/Feature/RequestsOptimizationTest.php | 74 +++++++++++++++++++ tests/Feature/SourceTest.php | 20 +++++ tests/Feature/StructureTest.php | 26 +++++++ tests/fixtures/pagination/page1.json | 4 +- tests/fixtures/pagination/page2.json | 4 +- tests/fixtures/pagination/page3.json | 4 +- .../fixtures/paginationWith5Pages/page1.json | 27 +++++++ .../fixtures/paginationWith5Pages/page2.json | 27 +++++++ .../fixtures/paginationWith5Pages/page3.json | 27 +++++++ .../fixtures/paginationWith5Pages/page4.json | 27 +++++++ .../fixtures/paginationWith5Pages/page5.json | 24 ++++++ 16 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/paginationWith5Pages/page1.json create mode 100644 tests/fixtures/paginationWith5Pages/page2.json create mode 100644 tests/fixtures/paginationWith5Pages/page3.json create mode 100644 tests/fixtures/paginationWith5Pages/page4.json create mode 100644 tests/fixtures/paginationWith5Pages/page5.json diff --git a/src/Concerns/RespectsRateLimits.php b/src/Concerns/RespectsRateLimits.php index 6b3b000..2088c75 100644 --- a/src/Concerns/RespectsRateLimits.php +++ b/src/Concerns/RespectsRateLimits.php @@ -4,6 +4,8 @@ namespace Cerbero\LazyJsonPages\Concerns; +use Cerbero\LazyJsonPages\Services\ClientFactory; + /** * The trait to respect rate limits of APIs. */ @@ -14,8 +16,8 @@ trait RespectsRateLimits */ protected function respectRateLimits(): void { - if (microtime(true) < $timestamp = $this->config->rateLimits?->resetAt() ?? 0) { - time_sleep_until($timestamp); + if (microtime(true) < $timestamp = $this->config->rateLimits?->resetAt() ?? 0.0) { + ClientFactory::isFake() ? ClientFactory::$fakedRateLimits[] = $timestamp : time_sleep_until($timestamp); } } } diff --git a/src/Paginations/Pagination.php b/src/Paginations/Pagination.php index 81be2eb..44b23a8 100644 --- a/src/Paginations/Pagination.php +++ b/src/Paginations/Pagination.php @@ -48,6 +48,8 @@ final public function __construct( /** * Determine whether this pagination matches the configuration. + * + * @codeCoverageIgnore */ public function matches(): bool { diff --git a/src/Services/ClientFactory.php b/src/Services/ClientFactory.php index 142a9ff..4af9845 100644 --- a/src/Services/ClientFactory.php +++ b/src/Services/ClientFactory.php @@ -42,6 +42,18 @@ final class ClientFactory */ private static array $globalMiddleware = []; + /** + * Whether HTTP requests are faked. + */ + private static bool $isFake = false; + + /** + * The faked rate limits timestamps. + * + * @var float[] + */ + public static array $fakedRateLimits = []; + /** * The tap middleware callbacks. */ @@ -84,6 +96,8 @@ public static function fake(array $responses, Closure $callback): array { $transactions = []; + self::$isFake = true; + $handler = HandlerStack::create(new MockHandler($responses)); $handler->push(Middleware::history($transactions)); @@ -94,9 +108,21 @@ public static function fake(array $responses, Closure $callback): array unset(self::$defaultConfig['handler']); + self::$fakedRateLimits = []; + + self::$isFake = false; + return $transactions; } + /** + * Determine whether the HTTP requests are faked. + */ + public static function isFake(): bool + { + return self::$isFake; + } + /** * Instantiate the class. */ diff --git a/src/Sources/Source.php b/src/Sources/Source.php index 21021ff..fc04ff7 100644 --- a/src/Sources/Source.php +++ b/src/Sources/Source.php @@ -39,6 +39,8 @@ final public function __construct( /** * Determine whether this class can handle the source. + * + * @codeCoverageIgnore */ public function matches(): bool { diff --git a/tests/Feature/PaginationTest.php b/tests/Feature/PaginationTest.php index 0d0a67b..43444dc 100644 --- a/tests/Feature/PaginationTest.php +++ b/tests/Feature/PaginationTest.php @@ -1,7 +1,11 @@ linkHeader() + ->collect('data.*'); + + $responses = [ + new Response(body: file_get_contents(fixture('pagination/page1.json'))), + ]; + + $transactions = ClientFactory::fake($responses, fn() => expect($lazyCollection)->sequence( + ['name' => 'item1'], + ['name' => 'item2'], + ['name' => 'item3'], + ['name' => 'item4'], + ['name' => 'item5'], + )); + + expect($transactions)->toHaveCount(1); + expect((string) $transactions[0]['request']->getUri())->toBe('https://example.com/api/v1/users'); +}); + it('fails if an invalid custom pagination is provided', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') ->pagination('Invalid') @@ -73,3 +98,19 @@ 'https://example.com/api/v1/users' => 'pagination/page1.json', ]); })->throws(InvalidPaginationException::class, 'The class [Invalid] should extend [Cerbero\LazyJsonPages\Paginations\Pagination].'); + +it('fails if an invalid JSON key is provided', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->totalPages('invalid') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + ]); +})->throws(InvalidKeyException::class, 'The key [invalid] does not contain a valid value.'); + +it('fails if a pagination is not supported', function () { + LazyJsonPages::from('https://example.com/api/v1/users') + ->collect('data.*') + ->each(fn() => true); +})->throws(UnsupportedPaginationException::class, 'The provided configuration does not match with any supported pagination.'); diff --git a/tests/Feature/RequestsOptimizationTest.php b/tests/Feature/RequestsOptimizationTest.php index b7d0832..5134158 100644 --- a/tests/Feature/RequestsOptimizationTest.php +++ b/tests/Feature/RequestsOptimizationTest.php @@ -1,7 +1,13 @@ sequence('sending', 'sent', 'sending', 'sending', 'sent', 'sent'); }); + +it('handles failures when sending HTTP requests asynchronously', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->async(3) + ->attempts(3) + ->backoff(fn() => 0) + ->totalPages('meta.total_pages') + ->collect('data.*'); + + $responses = [ + new Response(body: file_get_contents(fixture('paginationWith5Pages/page1.json'))), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + $e = new RequestException('connection failed', new Request('GET', 'https://example.com/api/v1/users?page=2')), + new Response(body: file_get_contents(fixture('paginationWith5Pages/page3.json'))), + ]; + + ClientFactory::fake($responses, function() use ($lazyCollection, $e) { + try { + $lazyCollection->toArray(); + } catch (Throwable $exception) { + expect($exception) + ->toBeInstanceOf(OutOfAttemptsException::class) + ->getPrevious()->toBe($e) + ->failedPages->toContain(2) + ->items->pluck('name')->all()->toContain('item11', 'item12', 'item13', 'item14', 'item15'); + } + }); +}); + +it('sets the timeouts', function () { + LazyJsonPages::from('https://nonexisting.test') + ->totalPages('meta.total_pages') + ->connectionTimeout(0.01) + ->requestTimeout(0.01) + ->collect('data.*') + ->each(fn() => true); +})->throws(ConnectException::class, 'Connection refused for URI https://nonexisting.test'); + +it('respects rate limits', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->totalPages('meta.total_pages') + ->throttle(requests: 1, perSeconds: 1) + ->collect('data.*'); + + $responses = [ + new Response(body: file_get_contents(fixture('pagination/page1.json'))), + new Response(body: file_get_contents(fixture('pagination/page2.json'))), + new Response(body: file_get_contents(fixture('pagination/page3.json'))), + ]; + + $transactions = ClientFactory::fake($responses, function () use ($lazyCollection) { + expect($lazyCollection)->sequence(...require fixture('items.php')); + expect(ClientFactory::$fakedRateLimits)->toHaveCount(3); + }); + + $actualUris = array_map(fn(array $transaction) => (string) $transaction['request']->getUri(), $transactions); + + expect($actualUris)->toBe([ + 'https://example.com/api/v1/users', + 'https://example.com/api/v1/users?page=2', + 'https://example.com/api/v1/users?page=3', + ]); +}); diff --git a/tests/Feature/SourceTest.php b/tests/Feature/SourceTest.php index 09a7390..6dcc551 100644 --- a/tests/Feature/SourceTest.php +++ b/tests/Feature/SourceTest.php @@ -1,6 +1,10 @@ 'pagination/page3.json', ]); })->with('sources'); + +it('fails if a source is not supported', function () { + LazyJsonPages::from(123) + ->totalPages('total_pages') + ->collect('data.*') + ->each(fn() => true); +})->throws(UnsupportedSourceException::class, 'The provided source is not supported.'); + +it('fails if a Laravel client response did not send the request', function () { + $response = new Response(new Psr7Response(body: '{"cursor":"abc"}')); + + LazyJsonPages::from($response) + ->cursor('cursor') + ->collect('data.*') + ->each(fn() => true); +})->throws(RequestNotSentException::class, 'The source did not send any HTTP request.'); diff --git a/tests/Feature/StructureTest.php b/tests/Feature/StructureTest.php index 07452c7..db56c5f 100644 --- a/tests/Feature/StructureTest.php +++ b/tests/Feature/StructureTest.php @@ -16,6 +16,19 @@ ]); }); +it('supports paginations with the current page in the URI path and last page as a URI', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users/page/1') + ->pageInPath() + ->lastPage('meta.last_page_uri') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users/page/1' => 'pagination/page1.json', + 'https://example.com/api/v1/users/page/2' => 'pagination/page2.json', + 'https://example.com/api/v1/users/page/3' => 'pagination/page3.json', + ]); +}); + it('supports a custom pattern for paginations with the current page in the URI path', function () { $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users/page1') ->pageInPath('~/page(\d+)$~') @@ -92,3 +105,16 @@ 'https://example.com/api/v1/users?skip=10' => 'pagination/page3.json', ]); }); + +it('supports paginations with custom page name', function () { + $lazyCollection = LazyJsonPages::from('https://example.com/api/v1/users') + ->pageName('current_page') + ->totalPages('meta.total_pages') + ->collect('data.*'); + + expect($lazyCollection)->toLoadItemsViaRequests([ + 'https://example.com/api/v1/users' => 'pagination/page1.json', + 'https://example.com/api/v1/users?current_page=2' => 'pagination/page2.json', + 'https://example.com/api/v1/users?current_page=3' => 'pagination/page3.json', + ]); +}); diff --git a/tests/fixtures/pagination/page1.json b/tests/fixtures/pagination/page1.json index d483994..c6af2f7 100644 --- a/tests/fixtures/pagination/page1.json +++ b/tests/fixtures/pagination/page1.json @@ -20,6 +20,8 @@ "total_pages": 3, "total_items": 14, "last_page": 3, - "cursor": "cursor1" + "last_page_uri": "https://example.com/api/v1/users/page/3", + "cursor": "cursor1", + "cursor_uri": "https://example.com/api/v1/users/page/cursor1" } } diff --git a/tests/fixtures/pagination/page2.json b/tests/fixtures/pagination/page2.json index f592a12..f4b41e9 100644 --- a/tests/fixtures/pagination/page2.json +++ b/tests/fixtures/pagination/page2.json @@ -20,6 +20,8 @@ "total_pages": 3, "total_items": 14, "last_page": 3, - "cursor": "cursor2" + "last_page_uri": "https://example.com/api/v1/users/page/3", + "cursor": "cursor2", + "cursor_uri": "https://example.com/api/v1/users/page/cursor2" } } diff --git a/tests/fixtures/pagination/page3.json b/tests/fixtures/pagination/page3.json index ef375ac..b1e6712 100644 --- a/tests/fixtures/pagination/page3.json +++ b/tests/fixtures/pagination/page3.json @@ -17,6 +17,8 @@ "total_pages": 3, "total_items": 14, "last_page": 3, - "cursor": null + "last_page_uri": "https://example.com/api/v1/users/page/3", + "cursor": null, + "cursor_uri": null } } diff --git a/tests/fixtures/paginationWith5Pages/page1.json b/tests/fixtures/paginationWith5Pages/page1.json new file mode 100644 index 0000000..1b82365 --- /dev/null +++ b/tests/fixtures/paginationWith5Pages/page1.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "name": "item1" + }, + { + "name": "item2" + }, + { + "name": "item3" + }, + { + "name": "item4" + }, + { + "name": "item5" + } + ], + "meta": { + "total_pages": 5, + "total_items": 24, + "last_page": 5, + "last_page_uri": "https://example.com/api/v1/users/page/5", + "cursor": "cursor1", + "cursor_uri": "https://example.com/api/v1/users/page/cursor1" + } +} diff --git a/tests/fixtures/paginationWith5Pages/page2.json b/tests/fixtures/paginationWith5Pages/page2.json new file mode 100644 index 0000000..f4539f3 --- /dev/null +++ b/tests/fixtures/paginationWith5Pages/page2.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "name": "item6" + }, + { + "name": "item7" + }, + { + "name": "item8" + }, + { + "name": "item9" + }, + { + "name": "item10" + } + ], + "meta": { + "total_pages": 5, + "total_items": 24, + "last_page": 5, + "last_page_uri": "https://example.com/api/v1/users/page/5", + "cursor": "cursor2", + "cursor_uri": "https://example.com/api/v1/users/page/cursor2" + } +} diff --git a/tests/fixtures/paginationWith5Pages/page3.json b/tests/fixtures/paginationWith5Pages/page3.json new file mode 100644 index 0000000..88a3c4b --- /dev/null +++ b/tests/fixtures/paginationWith5Pages/page3.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "name": "item11" + }, + { + "name": "item12" + }, + { + "name": "item13" + }, + { + "name": "item14" + }, + { + "name": "item15" + } + ], + "meta": { + "total_pages": 5, + "total_items": 24, + "last_page": 5, + "last_page_uri": "https://example.com/api/v1/users/page/5", + "cursor": "cursor3", + "cursor_uri": "https://example.com/api/v1/users/page/cursor3" + } +} diff --git a/tests/fixtures/paginationWith5Pages/page4.json b/tests/fixtures/paginationWith5Pages/page4.json new file mode 100644 index 0000000..2370ce2 --- /dev/null +++ b/tests/fixtures/paginationWith5Pages/page4.json @@ -0,0 +1,27 @@ +{ + "data": [ + { + "name": "item16" + }, + { + "name": "item17" + }, + { + "name": "item18" + }, + { + "name": "item19" + }, + { + "name": "item20" + } + ], + "meta": { + "total_pages": 5, + "total_items": 24, + "last_page": 5, + "last_page_uri": "https://example.com/api/v1/users/page/5", + "cursor": "cursor4", + "cursor_uri": "https://example.com/api/v1/users/page/cursor4" + } +} diff --git a/tests/fixtures/paginationWith5Pages/page5.json b/tests/fixtures/paginationWith5Pages/page5.json new file mode 100644 index 0000000..2d28813 --- /dev/null +++ b/tests/fixtures/paginationWith5Pages/page5.json @@ -0,0 +1,24 @@ +{ + "data": [ + { + "name": "item21" + }, + { + "name": "item22" + }, + { + "name": "item23" + }, + { + "name": "item24" + } + ], + "meta": { + "total_pages": 5, + "total_items": 24, + "last_page": 5, + "last_page_uri": "https://example.com/api/v1/users/page/5", + "cursor": null, + "cursor_uri": null + } +} From e9200f7210f3d3a00ccbf0748c76f3988ce82018 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 10 Sep 2024 20:13:42 -0300 Subject: [PATCH 107/109] Update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5178936..0298050 100644 --- a/README.md +++ b/README.md @@ -343,8 +343,8 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; LazyJsonPages::from($source) - ->onRequest(fn (RequestInterface $request) => ...) - ->onResponse(fn (ResponseInterface $response, RequestInterface $request) => ...); + ->onRequest(fn(RequestInterface $request) => ...) + ->onResponse(fn(ResponseInterface $response, RequestInterface $request) => ...); ``` We can also tweak the number of allowed seconds before an API connection times out or the allowed duration of the entire HTTP request (by default they are both set to 5 seconds): @@ -361,7 +361,7 @@ If the 3rd party API is faulty or error-prone, we can indicate how many times we // repeat failing requests 5 times after a backoff of 1, 2, 3, 4 and 5 seconds LazyJsonPages::from($source) ->attempts(5) - ->backoff(fn (int $attempt) => $attempt * 1000); + ->backoff(fn(int $attempt) => $attempt * 1000); ``` ### ๐Ÿ’ข Errors handling @@ -373,7 +373,7 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; LazyJsonPages::from($source) - ->onError(fn (Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); + ->onError(fn(Throwable $e, RequestInterface $request, ?ResponseInterface $response) => ...); ``` Any exception thrown by this package extends the `LazyJsonPagesException` class. This makes it easy to handle all exceptions in a single catch block: From c7cbb4a4863a75a3d43abdf836a00b6ddea20430 Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 10 Sep 2024 20:36:15 -0300 Subject: [PATCH 108/109] Remove example --- tests/Feature/ExampleTest.php | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 tests/Feature/ExampleTest.php diff --git a/tests/Feature/ExampleTest.php b/tests/Feature/ExampleTest.php deleted file mode 100644 index b27671c..0000000 --- a/tests/Feature/ExampleTest.php +++ /dev/null @@ -1,5 +0,0 @@ -expect(true) - ->toBe(true); From 00d57dba5fb4ebfb96ee1fb052f168f9edd3a00d Mon Sep 17 00:00:00 2001 From: Andrea Marco Sartori Date: Tue, 10 Sep 2024 20:52:28 -0300 Subject: [PATCH 109/109] Update changelog --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 234f4e8..d391ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ All notable changes to `lazy-json-pages` will be documented in this file. Updates should follow the [Keep a CHANGELOG](https://keepachangelog.com/) principles. +## 2.0.0 - 2024-09-10 + +### Added +- Dependency on Lazy JSON v2 +- Support for page numbers in URLs +- Custom Guzzle handler +- Auto-registered macro for lazy collections +- Support for offset +- Support for limit +- Support for Link header +- Support for API rate limits +- Support for Laravel events + +### Changed +- The path is now part of the configuration +- The retry logic + +### Removed +- Support for PHP versions older than 8.1 +- Support for configurations different from the Config DTO + +### Fixed +- Build badge on readme + + ## 1.0.0 - 2021-08-08 ### Added