Skip to content

Commit

Permalink
Merge pull request #146 from bluecadet/fix/json-media
Browse files Browse the repository at this point in the history
fix json media url, and update get-urls
  • Loading branch information
claytercek authored Jun 18, 2024
2 parents fc10c76 + c203bda commit 049642c
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 5 deletions.
5 changes: 5 additions & 0 deletions .changeset/shy-pears-sleep.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@bluecadet/launchpad-content": patch
---

Fix json content source media URL scraping
3 changes: 1 addition & 2 deletions packages/content/lib/content-sources/json-source.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ class JsonSource extends ContentSource {
*/
async _downloadJsons() {
const result = new ContentResult();

for (const [path, url] of Object.entries(this.config.files)) {
this.logger.debug(`Downloading json ${chalk.blue(url)}`);
const response = await got(url);
Expand All @@ -74,7 +73,7 @@ class JsonSource extends ContentSource {
for (const dataFile of result.dataFiles) {
this.logger.debug(`Scraping for media files in ${chalk.blue(dataFile.localPath)}...`);
const mediaUrls = JsonUtils.getUrls(dataFile.content, undefined, this.config.mediaPattern);
this.logger.debug(`Found ${chalk.blue(mediaUrls.size)} media files in ${chalk.blue(dataFile.localPath)}`);
this.logger.debug(`Found ${chalk.blue(mediaUrls.size.toString())} media files in ${chalk.blue(dataFile.localPath)}`);
result.addMediaDownloads([...mediaUrls].map(url => new MediaDownload({ url })));
}
return result;
Expand Down
7 changes: 5 additions & 2 deletions packages/content/lib/utils/json-utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class JsonUtils {
}

const possibleUrls = getUrls(JSON.stringify(json), config);

for (const url of possibleUrls) {
if (include && (include instanceof RegExp) && !url.match(include)) {
continue; // url doesn't match include
Expand All @@ -35,15 +36,17 @@ class JsonUtils {
}

/**
* @param {any} [userSettings]
* @param {import('get-urls').Options} [userSettings]
* @returns {import('get-urls').Options}
*/
static getUrlOptions(userSettings) {
return {
...{
stripAuthentication: false,
stripWWW: false,
removeTrailingSlash: false,
sortQueryParameters: false
sortQueryParameters: false,
requireSchemeOrWww: true
},
...userSettings
};
Expand Down
3 changes: 2 additions & 1 deletion packages/content/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"contentful": "^9.0.0",
"filenamify": "^5.1.1",
"fs-extra": "^10.0.0",
"get-urls": "^10.0.0",
"get-urls": "^12.1.0",
"got": "^12.3.0",
"jsonpath": "^1.1.1",
"markdown-it": "^12.2.0",
Expand All @@ -60,6 +60,7 @@
"devDependencies": {
"@types/cli-progress": "^3.11.0",
"@types/jsonpath": "^0.2.0",
"@types/markdown-it": "^12.2.0",
"@types/progress-stream": "^2.0.2",
"@types/qs": "^6.9.7",
"@types/rimraf": "^4.0.5",
Expand Down

0 comments on commit 049642c

Please sign in to comment.