diff --git a/docs/concepts/async-paging.md b/docs/concepts/async-paging.md new file mode 100644 index 000000000..55fa4ef4f --- /dev/null +++ b/docs/concepts/async-paging.md @@ -0,0 +1,110 @@ +# Async Paging + +With the introduction of the async iterator pattern to both sp/items and all graph collections we wanted to share a discussion for working with async paging. + +The easiest example is to process all of the items in a loop. In this example each page of 1000 results is retrieved from the list. The `items` collection itself is AsyncIterable so you can use it directly in the loop. + +```TypeScript +for await (const items of sp.web.lists.getByTitle("BigList").items.top(1000)) { + console.log(items.length); +} +``` + +And a graph example: + +```TypeScript +for await (const items of graph.users) { + console.log(items.length); +} +``` + +## Accessing the Iterator + +You might in some cases want to access the iterator object directly from the collection, which you can do using `Symbol.asyncIterator` method: + +```TypeScript +const iterator = collection[Symbol.asyncIterator](); +``` + +## Paging Helper + +We are also providing an example paging class to control prev/next paging through the collection using the AsyncIterator. The code here is provided as an example only. + +```TypeScript +class AsyncPager { + + private iterator: AsyncIterator; + + constructor(iterable: AsyncIterable, private pages: T[] = [], private pagePointer = -1, private isDone = false) { + this.iterator = iterable[Symbol.asyncIterator](); + } + + /** + * Provides access to the current page of values + */ + async current(): Promise { + + // we don't have any pages yet + if (this.pagePointer < 0) { + return this.next(); + } + + // return the current page + return this.pages[this.pagePointer]; + } + + /** + * Access the next page, either from the local cache or make a request to load it + */ + async next(): Promise { + + // does the page exist? + let page = this.pages[++this.pagePointer]; + + if (typeof page === "undefined") { + + if (this.isDone) { + + // if we are already done make sure we don't make any more requests + // and return the last page + --this.pagePointer; + + } else { + + // get the next page of links + const next = await this.iterator.next(); + + if (next.done) { + + this.isDone = true; + + } else { + + this.pages.push(next.value); + } + } + } + + return this.pages[this.pagePointer]; + } + + async prev(): Promise { + + // handle already at the start + if (this.pagePointer < 1) { + return this.pages[0]; + } + + // return the previous page moving our pointer + return this.pages[--this.pagePointer]; + } +} +``` + +And some usage: + +```TypeScript +const pager = new AsyncPager(sp.web.lists.getByTitle("BigList").items.top(1000)); + +const items1 = await pager.next(); +``` diff --git a/mkdocs.yml b/mkdocs.yml index c56dd035d..67db74852 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - '2020': 'news/2020-year-in-review.md' - 'Library Usage Tips': - 'Authentication': 'concepts/authentication.md' + - 'Async Paging': 'concepts/async-paging.md' - 'In SPFx': 'concepts/auth-spfx.md' - 'In Browser': 'concepts/auth-browser.md' - 'In NodeJS': 'concepts/auth-nodejs.md' diff --git a/packages/graph/package.json b/packages/graph/package.json index e7f0fc6b2..434379db7 100644 --- a/packages/graph/package.json +++ b/packages/graph/package.json @@ -4,9 +4,6 @@ "description": "pnp - provides a fluent interface to query the Microsoft Graph", "main": "./index.js", "typings": "./index", - "scripts": { - "postinstall": "node ./post-install.cjs" - }, "dependencies": { "@microsoft/microsoft-graph-types": "2.40.0", "tslib": "2.6.2", diff --git a/packages/graph/post-install.cjs b/packages/graph/post-install.cjs deleted file mode 100644 index 2686f800d..000000000 --- a/packages/graph/post-install.cjs +++ /dev/null @@ -1,20 +0,0 @@ -const { readFileSync } = require("fs"); -const { join } = require("path"); -const projectRoot = process.cwd(); -const packageLoc = join(projectRoot, "package.json"); -const packageFile = readFileSync(packageLoc, "utf8"); -const packageJSON = JSON.parse(packageFile); -if (packageJSON.dependencies != null) { - const spfxVersion = packageJSON.dependencies["@microsoft/sp-core-library"]; - if (spfxVersion != null) { - const spfxVersionFloat = parseFloat(spfxVersion); - if (spfxVersionFloat > 1.11 && spfxVersionFloat < 1.15) { - console.log(""); - console.log("\x1b[43m%s\x1b[0m", " PnPjs WARNING "); - console.log("\x1b[33m%s\x1b[0m", " The version of SPFx you are using requires an update to work with PnPjs. Please make sure to follow the getting started instructions to make the appropriate changes. ➡ https://pnp.github.io/pnpjs/getting-started/#spfx-version-1121-later"); - console.log(""); - } - } -} else { - console.log("Package has no dependencies"); -} diff --git a/packages/queryable/behaviors/caching.ts b/packages/queryable/behaviors/caching.ts index f1dad8446..a870fae8d 100644 --- a/packages/queryable/behaviors/caching.ts +++ b/packages/queryable/behaviors/caching.ts @@ -103,6 +103,7 @@ export function Caching(props?: ICachingProps): TimelinePipe { })); } else { + result = cached; } } diff --git a/packages/sp/files/types.ts b/packages/sp/files/types.ts index 020d6e9cf..3a50d27cb 100644 --- a/packages/sp/files/types.ts +++ b/packages/sp/files/types.ts @@ -738,6 +738,9 @@ function applyChunckedOperationDefaults(props: Partial): }; } +/** + * Converts the source into a ReadableStream we can understand + */ function sourceToReadableStream(source: ValidFileContentSource): ReadableStream { if (isBlob(source)) { @@ -763,7 +766,7 @@ function sourceToReadableStream(source: ValidFileContentSource): ReadableStream } else { - return source; + return source; } } diff --git a/packages/sp/items/types.ts b/packages/sp/items/types.ts index a1c33fa35..e0c062905 100644 --- a/packages/sp/items/types.ts +++ b/packages/sp/items/types.ts @@ -62,14 +62,14 @@ export class _Items extends _SPCollection { public [Symbol.asyncIterator]() { - const q = SPCollection(this).using(parseBinderWithErrorCheck(async (r) => { + const nextInit = SPCollection(this).using(parseBinderWithErrorCheck(async (r) => { const json = await r.json(); - const nextUrl = hOP(json, "d") && hOP(json.d, "__next") ? json.d.__next : json["odata.nextLink"]; + const nextLink = hOP(json, "d") && hOP(json.d, "__next") ? json.d.__next : json["odata.nextLink"]; return >{ - hasNext: typeof nextUrl === "string" && nextUrl.length > 0, - nextLink: nextUrl, + hasNext: typeof nextLink === "string" && nextLink.length > 0, + nextLink, value: parseODataJSON(json), }; })); @@ -79,13 +79,13 @@ export class _Items extends _SPCollection { for (let i = 0; i < queryParams.length; i++) { const param = this.query.get(queryParams[i]); if (objectDefinedNotNull(param)) { - q.query.set(queryParams[i], param); + nextInit.query.set(queryParams[i], param); } } return >{ - _next: q, + _next: nextInit, async next() { diff --git a/packages/sp/package.json b/packages/sp/package.json index e694c5597..4904d08df 100644 --- a/packages/sp/package.json +++ b/packages/sp/package.json @@ -4,9 +4,6 @@ "description": "pnp - provides a fluent api for working with SharePoint REST", "main": "./index.js", "typings": "./index", - "scripts": { - "postinstall": "node ./post-install.cjs" - }, "dependencies": { "tslib": "2.4.1", "@pnp/core": "0.0.0-PLACEHOLDER", diff --git a/packages/sp/post-install.cjs b/packages/sp/post-install.cjs deleted file mode 100644 index 2686f800d..000000000 --- a/packages/sp/post-install.cjs +++ /dev/null @@ -1,20 +0,0 @@ -const { readFileSync } = require("fs"); -const { join } = require("path"); -const projectRoot = process.cwd(); -const packageLoc = join(projectRoot, "package.json"); -const packageFile = readFileSync(packageLoc, "utf8"); -const packageJSON = JSON.parse(packageFile); -if (packageJSON.dependencies != null) { - const spfxVersion = packageJSON.dependencies["@microsoft/sp-core-library"]; - if (spfxVersion != null) { - const spfxVersionFloat = parseFloat(spfxVersion); - if (spfxVersionFloat > 1.11 && spfxVersionFloat < 1.15) { - console.log(""); - console.log("\x1b[43m%s\x1b[0m", " PnPjs WARNING "); - console.log("\x1b[33m%s\x1b[0m", " The version of SPFx you are using requires an update to work with PnPjs. Please make sure to follow the getting started instructions to make the appropriate changes. ➡ https://pnp.github.io/pnpjs/getting-started/#spfx-version-1121-later"); - console.log(""); - } - } -} else { - console.log("Package has no dependencies"); -}