From f53c15b635f1aab3b123fdc34f227ebfbb4f05ac Mon Sep 17 00:00:00 2001 From: Marcel Hoekstra Date: Sun, 8 Sep 2024 08:36:18 +0200 Subject: [PATCH] improved creating manifest create manifest and assessment works for 2.x as well. new commands to create package zips --- README.md | 20 +- package-lock.json | 4 +- package.json | 2 + scripts/build.ts | 4 +- .../qti-convert-cli/qti-package-per-item.ts | 17 + src/lib/qti-convert-cli/qti-package.ts | 17 + src/lib/qti-helper-node/qti-helper.ts | 487 +++++++++++++++--- src/lib/qti-helper/qti-strip.test.ts | 74 ++- 8 files changed, 546 insertions(+), 79 deletions(-) create mode 100644 src/lib/qti-convert-cli/qti-package-per-item.ts create mode 100644 src/lib/qti-convert-cli/qti-package.ts diff --git a/README.md b/README.md index 6849735..fc1195f 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Not only the files are remove but the reference in the item/test and manifest wi I you have a directory with one or more items but no assessment test, this command will create a assessment test that contains all the items that are in the foldername. It will override an existing assessment that's callled test.xml. -Should have the path to a folder as input parameter. This folder should contain the content of a qti3 package +Should have the path to a folder as input parameter. This folder should contain the content of a qti3 or qti2x package #### Creating or updating an manifest @@ -77,7 +77,23 @@ Should have the path to a folder as input parameter. This folder should contain ``` This will create or update an existing manifest. It will look into the directory and search for all items, tests and resources. -Also it will add the resources that are used in an item as a dependency. +Also it will add the resources that are used in an item as a dependency. This folder should contain the content of a qti3 or qti2x package + +#### Creating a package zip + +This create a package.zip based on all resources in a manifest. So it you have an existing package and you want to remove some items, you can extract the package.zip, remove the manifest, re-generate a manifest using qti-create-manifest and then run this command. The resources used in only the items you deleted, wont be packaged in the new zip. + +```sh + npx -p=@citolab/qti-convert qti-create-package yourpackage.zip +``` + +#### Creating a package zip per item + +This create a package.zip per item, for all items in a folder. The package will be called: package\_{item_title || item_identifer}.zip. + +```sh + npx -p=@citolab/qti-convert qti-create-package-per-item yourpackage.zip +``` ## API diff --git a/package-lock.json b/package-lock.json index a2db70e..2a4e782 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,9 @@ "qti-convert-pkg": "dist/qti-convert-pkg.mjs", "qti-create-assessment": "dist/qti-package-assessment.mjs", "qti-create-manifest": "dist/qti-package-manifest.mjs", - "qti-strip-media-pkg": "dist/qti-strip-media-pkg.mjs" + "qti-strip-media-pkg": "dist/qti-strip-media-pkg.mjs", + "qti-package": "dist/qti-package.mjs", + "qti-package-per-item": "dist/qti-package-per-item.mjs" }, "devDependencies": { "@arethetypeswrong/cli": "^0.13.2", diff --git a/package.json b/package.json index d3c2bb2..a9b5f2c 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,8 @@ "qti-convert-folder": "npx qti-convert-folder", "qti-package-manifest": "npx qti-create-manifest", "qti-package-assessment": "npx qti-create-assessment", + "qti-package": "npx qti-create-package", + "qti-package-per-item": "npx qti-create-package-per-item", "qti-strip-media-pkg": "npx qti-strip-media-pkg", "----dev----": "", "test": "vitest", diff --git a/scripts/build.ts b/scripts/build.ts index 9dd4b37..9616516 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -31,7 +31,9 @@ const cliOptions = { './src/lib/qti-convert-cli/qti-convert-folder.ts', './src/lib/qti-convert-cli/qti-package-manifest.ts', './src/lib/qti-convert-cli/qti-package-assessment.ts', - './src/lib/qti-convert-cli/qti-strip-media-pkg.ts' + './src/lib/qti-convert-cli/qti-strip-media-pkg.ts', + './src/lib/qti-convert-cli/qti-create-package.ts', + './src/lib/qti-convert-cli/qti-create-package-per-item.ts' ], splitting: true, bundle: true, diff --git a/src/lib/qti-convert-cli/qti-package-per-item.ts b/src/lib/qti-convert-cli/qti-package-per-item.ts new file mode 100644 index 0000000..6439a67 --- /dev/null +++ b/src/lib/qti-convert-cli/qti-package-per-item.ts @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { createPackageZipsPerItem } from '../qti-helper-node'; + +const folderLocation = process.argv[2]; + +if (!folderLocation) { + console.error('Please provide a folder location as an argument.'); + process.exit(1); +} + +try { + await createPackageZipsPerItem(folderLocation); + console.log('Done.'); +} catch (error) { + console.error(error); + process.exit(1); +} diff --git a/src/lib/qti-convert-cli/qti-package.ts b/src/lib/qti-convert-cli/qti-package.ts new file mode 100644 index 0000000..4d781b2 --- /dev/null +++ b/src/lib/qti-convert-cli/qti-package.ts @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { createPackageZip } from '../qti-helper-node'; + +const folderLocation = process.argv[2]; + +if (!folderLocation) { + console.error('Please provide a folder location as an argument.'); + process.exit(1); +} + +try { + await createPackageZip(folderLocation); + console.log('Done.'); +} catch (error) { + console.error(error); + process.exit(1); +} diff --git a/src/lib/qti-helper-node/qti-helper.ts b/src/lib/qti-helper-node/qti-helper.ts index 94216da..0048645 100644 --- a/src/lib/qti-helper-node/qti-helper.ts +++ b/src/lib/qti-helper-node/qti-helper.ts @@ -1,17 +1,67 @@ +import archiver from 'archiver'; import * as cheerio from 'cheerio'; -import { existsSync, lstatSync, readFileSync, readdirSync } from 'fs'; +import { createWriteStream, existsSync, lstatSync, readFileSync, readdirSync, writeFileSync } from 'fs'; +import { dirname } from 'path'; import xmlFormat from 'xml-formatter'; export const qtiReferenceAttributes = ['src', 'href', 'data', 'primary-path', 'fallback-path', 'template-location']; export type QtiResource = { - type: 'imsqti_test_xmlv3p0' | 'imsqti_item_xmlv3p0' | 'associatedcontent/learning-application-resource'; + type: + | 'imsqti_test_xmlv3p0' + | 'imsqti_item_xmlv3p0' + | 'associatedcontent/learning-application-resource' + | 'imsqti_item_xmlv2p2' + | 'imsqti_test_xmlv2p2' + | 'webcontent'; + href: string; identifier: string; dependencies: string[]; }; -export const getAllResourcesRecursively = (allResouces: QtiResource[], foldername: string) => { +export const determineQtiVersion = (foldername: string): '2.x' | '3.0' => { + // continue if the foldername is not a folder but a file + if (!lstatSync(foldername).isDirectory()) { + return undefined; + } + try { + const files = readdirSync(foldername); + for (const file of files) { + if (file === '.DS_Store') { + continue; + } + const subfolder = `${foldername}/${file}`; + if (lstatSync(subfolder).isDirectory()) { + const subResult = determineQtiVersion(subfolder); + if (subResult !== undefined) { + return subResult; + } + } else { + if (subfolder.endsWith('.xml')) { + const content = readFileSync(subfolder, 'utf-8'); + if (content.includes(' { + const assessmentTestTag = version === '2.x' ? 'assessmentTest' : 'qti-assessment-test'; + const assessmentItemTag = version === '2.x' ? 'assessmentItem' : 'qti-assessment-item'; // continue if the foldername is not a folder but a file if (!lstatSync(foldername).isDirectory()) { return; @@ -23,50 +73,36 @@ export const getAllResourcesRecursively = (allResouces: QtiResource[], foldernam continue; } const subfolder = `${foldername}/${file}`; - let processed = false; if (lstatSync(subfolder).isDirectory()) { - getAllResourcesRecursively(allResouces, subfolder); + getAllXmlResourcesRecursivelyWithDependencies(allResouces, subfolder, version); } else { if (subfolder.endsWith('.xml')) { - processed = true; const content = readFileSync(subfolder, 'utf-8'); - if (content.indexOf(' { +function getAttributeFromRawElement(element: cheerio.Element, attributeName: string): string | undefined { + // Check if the element has attributes and return the requested one if it exists + return element.attribs ? element.attribs[attributeName] : undefined; +} + +export const createPackageZipsPerItem = async (foldername: string) => { const manifest = `${foldername}/imsmanifest.xml`; - // check if manifest exists - const identfier = foldername.split('/').pop(); let manifestString = ''; + const version = await determineQtiVersion(foldername); + // Check if manifest exists if (!existsSync(manifest)) { - manifestString = ` + manifestString = await createOrCompleteManifest(foldername); + } else { + manifestString = readFileSync(manifest, 'utf-8'); + } + + const $manifestXml = cheerio.load(manifestString, { + xmlMode: true, + xml: true + }); + + // const allFiles = $manifestXml('file').toArray(); + // const packageFile = `${foldername}/package.zip`; + + const items: cheerio.Element[] = []; + + $manifestXml('resource').each((_, element) => { + const resourceType = $manifestXml(element).attr('type'); + if (resourceType && resourceType.includes('item')) { + items.push(element); + } + }); + const promises = []; + for (const item of items) { + const file = $manifestXml(item).attr('href'); + const itemContent = readFileSync(`${foldername}/${file}`, 'utf-8'); + const $item = cheerio.load(itemContent, { + xmlMode: true, + xml: true + }); + const title = $item(formatTagByVersion('assessment-item', version)).attr('title'); + const identfier = $item(formatTagByVersion('assessment-item', version)).attr('identifier'); + if (!title) { + throw new Error('Title is missing in ' + file); + } + const replaceAllCharsThatAreNotAllowedInaFileNameWithAUnderscore = (str: string) => + str.replace(/[^a-zA-Z0-9_]/g, '_'); + const packageFile = `${foldername}/package_${replaceAllCharsThatAreNotAllowedInaFileNameWithAUnderscore( + title || identfier + )}.zip`; + const $itemInManifest = $manifestXml(`resource[href="${file}"]`); + + const dependencies = $itemInManifest + .find('dependency') + .toArray() + .map(d => { + const ref = getAttributeFromRawElement(d, 'identifierref'); + const resource = $manifestXml(`resource[identifier="${ref}"]`); + const href = resource.attr('href'); + return href; + }) + .concat(file); + // Create a file stream for the zip output + const output = createWriteStream(packageFile); + + // Create the zip archive + const archive = archiver('zip', { + zlib: { level: 9 } // Compression level + }); + + // Pipe the archive data to the file stream + archive.pipe(output); + + // Handle archive finalization + promises.push( + new Promise((resolve, reject) => { + // Listen for errors + archive.on('error', err => { + console.log('Archiving error: ', err); + reject(err); + }); + + // When the output stream finishes, resolve the promise + output.on('close', () => { + console.log(`Archive has been finalized and the output file is ${archive.pointer()} total bytes`); + resolve(packageFile); + }); + // create a manifest file for this item. + const manifestString = getEmptyManifest(`manifest-${title}.xml`, version); + + // Append all files mentioned in the manifest to the archive + for (const href of dependencies) { + // const filename = href.split('/').pop(); + const fullPath = `${foldername}/${href}`; + archive.append(readFileSync(fullPath), { name: href }); + if (!existsSync(fullPath)) { + console.log(`Could not find the file: ${fullPath}`); + continue; + } + } + + const $itemManifest = cheerio.load(manifestString, { + xmlMode: true, + xml: true + }); + for (const href of dependencies) { + const elementInManifest = $manifestXml(`resource[href="${href}"]`); + if (elementInManifest.length > 0) { + $itemManifest('resources').append(elementInManifest); + } + } + const itemManifestString = $itemManifest.xml(); + archive.append(itemManifestString, { name: 'imsmanifest.xml' }); + + // Finalize the archive (no more files can be added after this point) + archive.finalize(); + }) + ); + } + return Promise.all(promises); +}; + +export const createPackageZip = async (foldername: string, createManifest = false, createAssessment = false) => { + const manifest = `${foldername}/imsmanifest.xml`; + + if (createAssessment) { + const assessmentFilename = `${foldername}/test.xml`; + const assessment = await createAssessmentTest(assessmentFilename); + writeFileSync(`${foldername}/test.xml`, assessment); + } + + if (createManifest) { + const manifestFilename = `${foldername}/imsmanifest.xml`; + const manifest = await createOrCompleteManifest(manifestFilename); + writeFileSync(`${foldername}/imsmanifest.xml`, manifest); + } + + // Check if manifest exists + if (!createManifest && !existsSync(manifest)) { + console.log(`Could not find the manifest file in the folder: ${foldername}`); + return; + } + + const manifestString = readFileSync(manifest, 'utf-8'); + const $manifestXml = cheerio.load(manifestString, { + xmlMode: true, + xml: true + }); + + const allFiles = $manifestXml('file').toArray(); + const packageFile = `${foldername}/package.zip`; + + // Create a file stream for the zip output + const output = createWriteStream(packageFile); + + // Create the zip archive + const archive = archiver('zip', { + zlib: { level: 9 } // Compression level + }); + + // Pipe the archive data to the file stream + archive.pipe(output); + + // Handle archive finalization + return new Promise((resolve, reject) => { + // Listen for errors + archive.on('error', err => { + console.log('Archiving error: ', err); + reject(err); + }); + + // When the output stream finishes, resolve the promise + output.on('close', () => { + console.log(`Archive has been finalized and the output file is ${archive.pointer()} total bytes`); + resolve(packageFile); + }); + + // Append all files mentioned in the manifest to the archive + for (const file of allFiles) { + const href = file.attribs.href; + // const filename = href.split('/').pop(); + const fullPath = `${foldername}/${href}`; + + if (!existsSync(fullPath)) { + console.log(`Could not find the file: ${fullPath}`); + continue; + } + + const fileContent = readFileSync(fullPath); + archive.append(fileContent, { name: href }); + } + // append the manifest self + archive.append(manifestString, { name: 'imsmanifest.xml' }); + // Finalize the archive (no more files can be added after this point) + archive.finalize(); + }); +}; + +const getEmptyManifest = (identifier: string, version: '2.x' | '3.0') => { + return version === '3.0' + ? ` QTI Package @@ -94,7 +324,28 @@ export const createOrCompleteManifest = async (foldername: string) => { - `; + ` + : ` + + + QTIv2.2 Package + 1.0.0 + + + + + `; +}; + +export const createOrCompleteManifest = async (foldername: string) => { + const manifest = `${foldername}/imsmanifest.xml`; + // check if manifest exists + const identfier = foldername.split('/').pop(); + const version = determineQtiVersion(foldername); + let manifestString = ''; + if (!existsSync(manifest)) { + manifestString = getEmptyManifest(identfier, version); } else { manifestString = readFileSync(manifest, 'utf-8'); } @@ -104,7 +355,53 @@ export const createOrCompleteManifest = async (foldername: string) => { }); const allResouces: QtiResource[] = []; - getAllResourcesRecursively(allResouces, foldername); + getAllXmlResourcesRecursivelyWithDependencies(allResouces, foldername, version); + + const allDependencies = allResouces.flatMap(r => + r.dependencies.map(d => ({ fileRef: d, referencedBy: r.identifier })) + ); + // Define the type for unique dependencies + type UniqueDependency = { + href: string; + id: string; + referencedBy: string[]; + }; + + // Create a new list of unique dependencies + const uniqueDependencies: UniqueDependency[] = []; + + allDependencies.forEach(dependency => { + const { fileRef, referencedBy } = dependency; + + // Extract the filename from href + const filename = fileRef.split('/').pop()?.replaceAll('.', '_') ?? ''; + + // Check if this dependency is already in the unique list + const existingDependency = uniqueDependencies.find(dep => dep.href === fileRef); + + if (existingDependency) { + // If it exists, add the new resource reference to the referencedBy array + if (!existingDependency.referencedBy.includes(referencedBy)) { + existingDependency.referencedBy.push(referencedBy); + } + } else { + // Create a new ID based on the uniqueness rules + let id = `RES-${filename}`; + + // If multiple resources reference the same filename but different hrefs + if (uniqueDependencies.some(dep => dep.id === id)) { + id = `RES-${referencedBy}-${filename}`; + } + + // Add the unique dependency + uniqueDependencies.push({ + href: fileRef, + id, + referencedBy: [referencedBy] + }); + } + }); + for (const resource of allResouces) { if ($manifestXml(`resource[identifier="${resource.identifier}"]`).length === 0) { const href = resource.href.replace(foldername, ''); @@ -117,19 +414,34 @@ export const createOrCompleteManifest = async (foldername: string) => { ); } if (resource.dependencies.length > 0) { + const dependencyFiles = uniqueDependencies.filter(d => d.referencedBy.includes(resource.identifier)); const manifestResource = $manifestXml(`resource[identifier="${resource.identifier}"]`); if (manifestResource.length > 0) { - for (const dependency of resource.dependencies) { - const dependencyNode = manifestResource.find(`dependency[identifierref="${dependency}"]`); + for (const dependency of dependencyFiles) { + const dependencyNode = manifestResource.find(`dependency[identifierref="${dependency.id}"]`); if (dependencyNode.length === 0) { // Append the dependency node if it doesn't exist - manifestResource.append(``); + manifestResource.append(``); } } } } } + for (const resource of uniqueDependencies) { + if ($manifestXml(`resource[identifier="${resource.id}"]`).length === 0) { + const href = resource.href.replace(foldername, ''); + // remove first slash if it exists + const hrefWithoutLeadingSlash = href[0] === '/' ? href.slice(1) : href; + $manifestXml('resources').append( + ` + + ` + ); + } + } let xmlString = $manifestXml.xml(); // Remove the BOM character if it exists: https://github.com/cheeriojs/cheerio/issues/1117 if (xmlString.startsWith('')) { @@ -143,40 +455,81 @@ export const createOrCompleteManifest = async (foldername: string) => { return formattedXML; }; +const formatTagByVersion = (tagName: string, version: '3.0' | '2.x') => { + if (version === '3.0') { + return `qti-${tagName}`; + } else { + // Convert to camelCase, remove the "qti-" prefix, and return the formatted tag + return tagName + .replace(/-./g, x => x[1].toUpperCase()) // Converts dash-case to camelCase + .replace(/^qti/, ''); // Remove qti- prefix if present + } +}; + +const formatAttributesByVersion = (attributes: string, version: '3.0' | '2.x') => { + if (version === '3.0') { + return attributes; + } else { + return attributes.replace(/([a-z])-([a-z])/g, (_, p1, p2) => p1 + p2.toUpperCase()); + } +}; + export const createAssessmentTest = async (foldername: string) => { const allResouces: QtiResource[] = []; - getAllResourcesRecursively(allResouces, foldername); - const items = allResouces.filter(item => item.type === 'imsqti_item_xmlv3p0'); + const version = determineQtiVersion(foldername); + getAllXmlResourcesRecursivelyWithDependencies(allResouces, foldername, version); + const items = allResouces.filter(item => item.type.includes('imsqti_item')); + + const formatTag = (tagName: string) => { + return formatTagByVersion(tagName, version); + }; + + const formatAttributes = (attributes: string) => { + return formatAttributesByVersion(attributes, version); + }; const xmlString = ` - - - - 0.0 - - - - + title="My Test" tool-name="Spectatus" identifier="TST-GENERATED-TEST">` + : `` +} + <${formatTag('outcome-declaration')} ${formatAttributes( + 'base-type="float" cardinality="single" identifier="SCORE"' + )}> + <${formatTag('default-value')}> + <${formatTag('value')}>0.0 + + + <${formatTag('test-part')} ${formatAttributes( + 'submission-mode="simultaneous" navigation-mode="nonlinear" identifier="TP"' + )}> + <${formatTag('assessment-section')} title="Section 1" visible="true" identifier="S1"> ${items .map(item => { const relativePath = item.href.replace(foldername + '/', '').replace(foldername, ''); - return ``; + return `<${formatTag('assessment-item-ref')} href="${relativePath}" identifier="${item.identifier}"/>`; }) .join('\n')} - - - - - - - - - -`; + + + <${formatTag('outcome-processing')}> + <${formatTag('set-outcome-value')} identifier="SCORE"> + <${formatTag('sum')}> + <${formatTag('test-variables')} ${formatAttributes('base-type="float" variable-identifier="SCORE"')}/> + + + +`; const formattedXML = xmlFormat(xmlString, { indentation: ' ', collapseContent: true, @@ -185,29 +538,29 @@ export const createAssessmentTest = async (foldername: string) => { return formattedXML; }; -const getDependencies = ($: cheerio.CheerioAPI) => { - const identifiers = []; +const getDependencyReferences = (pathPath: string, $: cheerio.CheerioAPI, version: '2.x' | '3.0') => { + const filenames = []; // Get qti-assessment-item identifiers - $('qti-assessment-item-ref').each((i, elem) => { + $(formatTagByVersion('assessment-item-ref', version)).each((i, elem) => { const identifier = $(elem).attr('identifier'); if (identifier) { - identifiers.push(identifier); + filenames.push(identifier); } }); qtiReferenceAttributes.forEach(selector => { $(`[${selector}]`).each((i, elem) => { - if (elem.type !== 'tag' || elem.name !== 'qti-assessment-item-ref') { + if (elem.type !== 'tag' || elem.name !== formatTagByVersion('assessment-item-ref', version)) { const attr = $(elem).attr(selector); if (attr) { - const filename = attr.split('/').pop(); - const identifier = `RES-${filename.replace(/\./g, '_')}`; - identifiers.push(identifier); + const directoryName = dirname(pathPath); + const filename = `${directoryName}/${attr}`; + filenames.push(filename); } } }); }); - return identifiers; + return filenames; }; diff --git a/src/lib/qti-helper/qti-strip.test.ts b/src/lib/qti-helper/qti-strip.test.ts index c7c914e..73f455f 100644 --- a/src/lib/qti-helper/qti-strip.test.ts +++ b/src/lib/qti-helper/qti-strip.test.ts @@ -1,16 +1,74 @@ // import { readFileSync, writeFile } from 'fs'; // import { removeMediaFromPackage } from './qti-helper'; import { expect, test } from 'vitest'; +// import { +// createAssessmentTest, +// createOrCompleteManifest, +// createPackageZip, +// createPackageZipsPerItem +// } from '../qti-helper-node'; +// import { writeFileSync } from 'fs'; + +// const pkg = ''; + +// test( +// 'strip media', +// async () => { +// // const pkg = `/`; +// // const outputFileName = pkg.replace('.zip', '-stripped.zip'); +// // const file = readFileSync(pkg); +// // const blob = await removeMediaFromPackage(file); +// // const buffer = Buffer.from(await blob.arrayBuffer()); + +// // await writeFile(outputFileName, buffer, () => 'Successfully converted the package: ' + outputFileName + '.'); +// expect(false).toEqual(false); +// }, +// { timeout: 100000 } +// ); + +// test( +// 'create test', +// async () => { +// const assessment = await createAssessmentTest(pkg); +// writeFileSync(`${pkg}/test.xml`, assessment); +// console.log('Successfully added/completed the test.'); +// expect(false).toEqual(false); +// }, +// { timeout: 100000 } +// ); + +// test( +// 'create manifest', +// async () => { +// const manifest = await createOrCompleteManifest(pkg); +// writeFileSync(`${pkg}/imsmanifest.xml`, manifest); +// console.log('Successfully added/completed the manifest.'); +// expect(false).toEqual(false); +// }, +// { timeout: 100000 } +// ); + +// test( +// 'create package', +// async () => { +// await createPackageZip(pkg); +// expect(false).toEqual(false); +// }, +// { timeout: 100000 } +// ); + +// test( +// 'create package per item', +// async () => { +// await createPackageZipsPerItem(pkg); +// expect(false).toEqual(false); +// }, +// { timeout: 100000 } +// ); + test( - 'strip media', + 'test file to test', async () => { - // const pkg = `/`; - // const outputFileName = pkg.replace('.zip', '-stripped.zip'); - // const file = readFileSync(pkg); - // const blob = await removeMediaFromPackage(file); - // const buffer = Buffer.from(await blob.arrayBuffer()); - - // await writeFile(outputFileName, buffer, () => 'Successfully converted the package: ' + outputFileName + '.'); expect(false).toEqual(false); }, { timeout: 100000 }