From 69e656221cd4fc596883a79d732a19f8b870e1c1 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 7 Sep 2024 12:04:33 +0100 Subject: [PATCH 1/8] Generate version specifc individual software pages --- .../software-markdown-pages.py | 90 +++++++++++++++++++ .../update-version-specific-docs.sh | 14 ++- mkdocs.yml | 4 +- 3 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 docs/version-specific/software-markdown-pages.py diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py new file mode 100644 index 000000000..fa493b402 --- /dev/null +++ b/docs/version-specific/software-markdown-pages.py @@ -0,0 +1,90 @@ +import argparse +import json +import shutil +from collections import defaultdict +from pathlib import Path + + +def output_markdown(processed, output_base_path): + packages = sum(len(v) for v in processed.values()) + top_page = open(output_base_path / 'index.md', 'w') + top_page.write("# List of supported software\n\n") + top_page.write(f"EasyBuild supports {packages} different software packages (incl. toolchains, bundles):\n\n") + + for letter in processed: + top_page.write(f" * [{letter}]({letter}/index.md)\n") + + letter_dir = output_base_path / letter + letter_dir.mkdir() + letter_page = open(letter_dir / 'index.md', 'w') + letter_page.write(f"# List of supported software ({letter})\n\n") + + for software in processed[letter]: + top_page.write(f" * [{software}]({letter}/{software}.md)\n") + letter_page.write(f" * [{software}]({software}.md)\n") + + versionsuffix = any(v['versionsuffix'] for v in processed[letter][software]) + + software_page = open(letter_dir / f'{software}.md', 'w') + software_page.write(f"# {software}\n\n") + software_page.write(f"{processed[letter][software][0]['description']}\n\n") + software_page.write(f"*homepage*: <{processed[letter][software][0]['homepage']}>\n\n") + + if versionsuffix: + software_page.write("version | versionsuffix | toolchain\n") + software_page.write("--------|---------------|----------\n") + else: + software_page.write("version | toolchain\n") + software_page.write("--------|----------\n") + + for version in processed[letter][software]: + software_page.write(f"``{version['version']}`` | ") + if versionsuffix: + if version['versionsuffix']: + software_page.write(f"``{version['versionsuffix']}``") + software_page.write(" | ") + software_page.write(f"``{version['toolchain']}``\n") + + software_page.close() + + letter_page.close() + + top_page.close() + + +def generate_markdown_pages(jsonfile, output_base, delete_existing): + """ + Generate markdown + :param jsonfile: input file (json file) + :param output_base: base directory for output files + :param delete_existing: delete the output directory (if it exists) + """ + with open(jsonfile) as f: + data = json.load(f) + + processed = defaultdict(lambda: defaultdict(list)) + for software in data: + initial = software['name'][0].lower() + if initial.isnumeric(): + initial = '0' + processed[initial][software['name']].append(software) + + output_base_path = Path(output_base) + + if delete_existing and output_base_path.exists() and output_base_path.is_dir(): + shutil.rmtree(output_base_path) + + output_base_path.mkdir(parents=True) + output_markdown(processed, output_base_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog='Software Markdown Pages', + description='Generate Markdown pages of software from JSON file') + parser.add_argument('-j', '--jsonfile', default='software.json', help='Input JSON file') + parser.add_argument('-o', '--output-base', default='output', help='Base directory for output files') + parser.add_argument('--delete-existing-output', action='store_true', + help='Delete output base directory (if it exists)') + args = parser.parse_args() + + generate_markdown_pages(args.jsonfile, args.output_base, args.delete_existing_output) diff --git a/docs/version-specific/update-version-specific-docs.sh b/docs/version-specific/update-version-specific-docs.sh index 534b9cec9..17a7e0bf3 100755 --- a/docs/version-specific/update-version-specific-docs.sh +++ b/docs/version-specific/update-version-specific-docs.sh @@ -157,15 +157,11 @@ echo "* [List of known toolchains](toolchains.md)" >> $overview echo "eb --list-software=detailed" skip_lines='Temporary log file|Processed.*easyconfigs|Found.*different software packages|^# List of supported software' -echo "# List of supported software" > supported-software.md -echo >> supported-software.md -echo "!!! note" >> supported-software.md -echo >> supported-software.md -echo " This page contains a lot of information, it may take a while to load." >> supported-software.md -echo >> supported-software.md -eb --list-software=detailed --output-format=md | egrep -v $skip_lines >> supported-software.md - -echo "* [List of supported software](supported-software.md)" >> $overview +eb --list-software=detailed --output-format=json | egrep -v $skip_lines >> software.json +python software-markdown-pages.py -o supported-software --delete-existing-output +rm software.json + +echo "* [List of supported software](supported-software/index.md)" >> $overview ################################################################################################### diff --git a/mkdocs.yml b/mkdocs.yml index 418c48878..536d7e34a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -105,7 +105,7 @@ nav: - EasyBuild release notes: release-notes.md - EasyBuild maintainers: maintainers.md - API: api/ - - List of supported software: version-specific/supported-software.md + - List of supported software: version-specific/supported-software/index.md - Getting help: getting-help.md - User survey: user-survey/index.md - Policies: @@ -194,6 +194,8 @@ plugins: - api/easybuild/* - api/summary.md - index.md + - version-specific/supported-software/* + - version-specific/supported-software/*/* # necessary for search to work - search - redirects: From 93aa583d8415fddac2853ee7871179352dd71902 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 7 Sep 2024 13:50:55 +0100 Subject: [PATCH 2/8] down rank the pages in the search, add in inter page links --- .../software-markdown-pages.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index fa493b402..22c13f75a 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -5,9 +5,36 @@ from pathlib import Path +MKDOCS_SEARCH_PRIORITY = """--- +search: + boost: 0.5 +--- +""" + + +def generate_character_links_line(characters, current=None): + """ + Generate links to index page for each character + :param characters: Initial characters to generate links for + """ + links = [] + for c in characters: + if c == current: + links.append(f'*{c}*') + else: + links.append(f"[../{c}/index.md]({c})") + return f"{' - '.join(links)}\n\n" + + def output_markdown(processed, output_base_path): + """ + Output markdown pages (index and per letter directories, each with an index and page per software) + :param processed: Processed data to output (dictionary - letter -> software -> list of versions) + :param output_base_path: Pathlib object for base path of output + """ packages = sum(len(v) for v in processed.values()) top_page = open(output_base_path / 'index.md', 'w') + top_page.write(MKDOCS_SEARCH_PRIORITY) top_page.write("# List of supported software\n\n") top_page.write(f"EasyBuild supports {packages} different software packages (incl. toolchains, bundles):\n\n") @@ -17,7 +44,9 @@ def output_markdown(processed, output_base_path): letter_dir = output_base_path / letter letter_dir.mkdir() letter_page = open(letter_dir / 'index.md', 'w') + letter_page.write(MKDOCS_SEARCH_PRIORITY) letter_page.write(f"# List of supported software ({letter})\n\n") + letter_page.write(generate_character_links_line([v for v in processed], current=letter)) for software in processed[letter]: top_page.write(f" * [{software}]({letter}/{software}.md)\n") @@ -26,6 +55,7 @@ def output_markdown(processed, output_base_path): versionsuffix = any(v['versionsuffix'] for v in processed[letter][software]) software_page = open(letter_dir / f'{software}.md', 'w') + software_page.write(MKDOCS_SEARCH_PRIORITY) software_page.write(f"# {software}\n\n") software_page.write(f"{processed[letter][software][0]['description']}\n\n") software_page.write(f"*homepage*: <{processed[letter][software][0]['homepage']}>\n\n") @@ -45,6 +75,8 @@ def output_markdown(processed, output_base_path): software_page.write(" | ") software_page.write(f"``{version['toolchain']}``\n") + software_page.write('\n') + software_page.write(generate_character_links_line([v for v in processed])) software_page.close() letter_page.close() From d2231b0ee40066d1d2e261a58384cff85fe86849 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 7 Sep 2024 13:57:09 +0100 Subject: [PATCH 3/8] correction --- docs/version-specific/software-markdown-pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index 22c13f75a..0e6cc34b1 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -22,7 +22,7 @@ def generate_character_links_line(characters, current=None): if c == current: links.append(f'*{c}*') else: - links.append(f"[../{c}/index.md]({c})") + links.append(f"[{c}](../{c}/index.md)") return f"{' - '.join(links)}\n\n" From 79c120a3214810a6d75de231f6e78436b42befcd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 19:44:36 +0200 Subject: [PATCH 4/8] use None as default for --jsonfile option in software-markdown-pages.py --- docs/version-specific/software-markdown-pages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index 0e6cc34b1..6e77e23f7 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -1,6 +1,7 @@ import argparse import json import shutil +import sys from collections import defaultdict from pathlib import Path @@ -91,6 +92,10 @@ def generate_markdown_pages(jsonfile, output_base, delete_existing): :param output_base: base directory for output files :param delete_existing: delete the output directory (if it exists) """ + if jsonfile is None: + sys.stderr.write("ERROR: No input JSON file specified, it is required!\n") + sys.exit(1) + with open(jsonfile) as f: data = json.load(f) @@ -113,7 +118,7 @@ def generate_markdown_pages(jsonfile, output_base, delete_existing): if __name__ == "__main__": parser = argparse.ArgumentParser(prog='Software Markdown Pages', description='Generate Markdown pages of software from JSON file') - parser.add_argument('-j', '--jsonfile', default='software.json', help='Input JSON file') + parser.add_argument('-j', '--jsonfile', default=None, help='Input JSON file') parser.add_argument('-o', '--output-base', default='output', help='Base directory for output files') parser.add_argument('--delete-existing-output', action='store_true', help='Delete output base directory (if it exists)') From 8e15a347203391e61f9e296fd733e8d6e743ec42 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 19:45:03 +0200 Subject: [PATCH 5/8] use temporary file to catch overview of supported software in JSON format --- docs/version-specific/update-version-specific-docs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/version-specific/update-version-specific-docs.sh b/docs/version-specific/update-version-specific-docs.sh index 17a7e0bf3..be14eb47b 100755 --- a/docs/version-specific/update-version-specific-docs.sh +++ b/docs/version-specific/update-version-specific-docs.sh @@ -155,11 +155,12 @@ echo "* [List of known toolchains](toolchains.md)" >> $overview ################################################################################################### +tmp_software_json="$(mktemp)" echo "eb --list-software=detailed" skip_lines='Temporary log file|Processed.*easyconfigs|Found.*different software packages|^# List of supported software' -eb --list-software=detailed --output-format=json | egrep -v $skip_lines >> software.json -python software-markdown-pages.py -o supported-software --delete-existing-output -rm software.json +eb --list-software=detailed --output-format=json | egrep -v "$skip_lines" > ${tmp_software_json} +python3 software-markdown-pages.py --jsonfile ${tmp_software_json} --output-base supported-software --delete-existing-output +rm ${tmp_software_json} echo "* [List of supported software](supported-software/index.md)" >> $overview From c79e934db9bed86247923299e26c6110ef21ee5f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 19:57:39 +0200 Subject: [PATCH 6/8] consistently use context manager to write files in software-markdown-pages.py script --- .../software-markdown-pages.py | 94 +++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index 6e77e23f7..c52dc6f28 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -34,55 +34,51 @@ def output_markdown(processed, output_base_path): :param output_base_path: Pathlib object for base path of output """ packages = sum(len(v) for v in processed.values()) - top_page = open(output_base_path / 'index.md', 'w') - top_page.write(MKDOCS_SEARCH_PRIORITY) - top_page.write("# List of supported software\n\n") - top_page.write(f"EasyBuild supports {packages} different software packages (incl. toolchains, bundles):\n\n") - - for letter in processed: - top_page.write(f" * [{letter}]({letter}/index.md)\n") - - letter_dir = output_base_path / letter - letter_dir.mkdir() - letter_page = open(letter_dir / 'index.md', 'w') - letter_page.write(MKDOCS_SEARCH_PRIORITY) - letter_page.write(f"# List of supported software ({letter})\n\n") - letter_page.write(generate_character_links_line([v for v in processed], current=letter)) - - for software in processed[letter]: - top_page.write(f" * [{software}]({letter}/{software}.md)\n") - letter_page.write(f" * [{software}]({software}.md)\n") - - versionsuffix = any(v['versionsuffix'] for v in processed[letter][software]) - - software_page = open(letter_dir / f'{software}.md', 'w') - software_page.write(MKDOCS_SEARCH_PRIORITY) - software_page.write(f"# {software}\n\n") - software_page.write(f"{processed[letter][software][0]['description']}\n\n") - software_page.write(f"*homepage*: <{processed[letter][software][0]['homepage']}>\n\n") - - if versionsuffix: - software_page.write("version | versionsuffix | toolchain\n") - software_page.write("--------|---------------|----------\n") - else: - software_page.write("version | toolchain\n") - software_page.write("--------|----------\n") - - for version in processed[letter][software]: - software_page.write(f"``{version['version']}`` | ") - if versionsuffix: - if version['versionsuffix']: - software_page.write(f"``{version['versionsuffix']}``") - software_page.write(" | ") - software_page.write(f"``{version['toolchain']}``\n") - - software_page.write('\n') - software_page.write(generate_character_links_line([v for v in processed])) - software_page.close() - - letter_page.close() - - top_page.close() + + with open(output_base_path / 'index.md', 'w') as top_page: + top_page.write(MKDOCS_SEARCH_PRIORITY) + top_page.write("# List of supported software\n\n") + top_page.write(f"EasyBuild supports {packages} different software packages (incl. toolchains, bundles):\n\n") + + for letter in processed: + top_page.write(f" * [{letter}]({letter}/index.md)\n") + + letter_dir = output_base_path / letter + letter_dir.mkdir() + with open(letter_dir / 'index.md', 'w') as letter_page: + letter_page.write(MKDOCS_SEARCH_PRIORITY) + letter_page.write(f"# List of supported software ({letter})\n\n") + letter_page.write(generate_character_links_line([v for v in processed], current=letter)) + + for software in processed[letter]: + top_page.write(f" * [{software}]({letter}/{software}.md)\n") + letter_page.write(f" * [{software}]({software}.md)\n") + + versionsuffix = any(v['versionsuffix'] for v in processed[letter][software]) + + with open(letter_dir / f'{software}.md', 'w') as software_page: + software_page.write(MKDOCS_SEARCH_PRIORITY) + software_page.write(f"# {software}\n\n") + software_page.write(f"{processed[letter][software][0]['description']}\n\n") + software_page.write(f"*homepage*: <{processed[letter][software][0]['homepage']}>\n\n") + + if versionsuffix: + software_page.write("version | versionsuffix | toolchain\n") + software_page.write("--------|---------------|----------\n") + else: + software_page.write("version | toolchain\n") + software_page.write("--------|----------\n") + + for version in processed[letter][software]: + software_page.write(f"``{version['version']}`` | ") + if versionsuffix: + if version['versionsuffix']: + software_page.write(f"``{version['versionsuffix']}``") + software_page.write(" | ") + software_page.write(f"``{version['toolchain']}``\n") + + software_page.write('\n') + software_page.write(generate_character_links_line([v for v in processed])) def generate_markdown_pages(jsonfile, output_base, delete_existing): From a3e156e5cb436f90a099c1db6a0262f0f27e7bb0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 21:05:22 +0200 Subject: [PATCH 7/8] tweak quick links like + add it to top + bottom of software overview pages --- .../software-markdown-pages.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index c52dc6f28..b1c515714 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -13,18 +13,22 @@ """ -def generate_character_links_line(characters, current=None): +def generate_quick_links_line(chars, level, current=None): """ Generate links to index page for each character :param characters: Initial characters to generate links for """ + up = '/'.join(['..'] * level) or '.' links = [] - for c in characters: - if c == current: - links.append(f'*{c}*') + if level > 0: + links.append(f"[(all)]({up}/index.md)") + for char in chars: + if char == current: + links.append(f'*{char}*') else: - links.append(f"[{c}](../{c}/index.md)") - return f"{' - '.join(links)}\n\n" + links.append(f"[{char}]({up}/{char}/index.md)") + links_txt = ' - '.join(links) + return f"*(quick links: {links_txt})*\n\n" def output_markdown(processed, output_base_path): @@ -33,12 +37,14 @@ def output_markdown(processed, output_base_path): :param processed: Processed data to output (dictionary - letter -> software -> list of versions) :param output_base_path: Pathlib object for base path of output """ - packages = sum(len(v) for v in processed.values()) + software_cnt = sum(len(v) for v in processed.values()) + letters = sorted(processed.keys()) with open(output_base_path / 'index.md', 'w') as top_page: top_page.write(MKDOCS_SEARCH_PRIORITY) top_page.write("# List of supported software\n\n") - top_page.write(f"EasyBuild supports {packages} different software packages (incl. toolchains, bundles):\n\n") + top_page.write(generate_quick_links_line(letters, 0)) + top_page.write(f"EasyBuild supports {software_cnt} different software packages (incl. toolchains, bundles):\n\n") for letter in processed: top_page.write(f" * [{letter}]({letter}/index.md)\n") @@ -48,7 +54,7 @@ def output_markdown(processed, output_base_path): with open(letter_dir / 'index.md', 'w') as letter_page: letter_page.write(MKDOCS_SEARCH_PRIORITY) letter_page.write(f"# List of supported software ({letter})\n\n") - letter_page.write(generate_character_links_line([v for v in processed], current=letter)) + letter_page.write(generate_quick_links_line(letters, 1, current=letter) + "\n\n") for software in processed[letter]: top_page.write(f" * [{software}]({letter}/{software}.md)\n") @@ -77,8 +83,11 @@ def output_markdown(processed, output_base_path): software_page.write(" | ") software_page.write(f"``{version['toolchain']}``\n") - software_page.write('\n') - software_page.write(generate_character_links_line([v for v in processed])) + software_page.write("\n\n" + generate_quick_links_line(letters, 1)) + + letter_page.write("\n\n" + generate_quick_links_line(letters, 1, current=letter)) + + top_page.write("\n\n" + generate_quick_links_line(letters, 0)) def generate_markdown_pages(jsonfile, output_base, delete_existing): From ed7801aa57c74fdb7db9914019078f4f6fc6e56d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Sep 2024 21:15:52 +0200 Subject: [PATCH 8/8] fix long line in software-markdown-pages.py script --- docs/version-specific/software-markdown-pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/version-specific/software-markdown-pages.py b/docs/version-specific/software-markdown-pages.py index b1c515714..b1f19e1c5 100644 --- a/docs/version-specific/software-markdown-pages.py +++ b/docs/version-specific/software-markdown-pages.py @@ -37,14 +37,14 @@ def output_markdown(processed, output_base_path): :param processed: Processed data to output (dictionary - letter -> software -> list of versions) :param output_base_path: Pathlib object for base path of output """ - software_cnt = sum(len(v) for v in processed.values()) + pkg_cnt = sum(len(v) for v in processed.values()) letters = sorted(processed.keys()) with open(output_base_path / 'index.md', 'w') as top_page: top_page.write(MKDOCS_SEARCH_PRIORITY) top_page.write("# List of supported software\n\n") top_page.write(generate_quick_links_line(letters, 0)) - top_page.write(f"EasyBuild supports {software_cnt} different software packages (incl. toolchains, bundles):\n\n") + top_page.write(f"EasyBuild supports {pkg_cnt} different software packages (incl. toolchains, bundles):\n\n") for letter in processed: top_page.write(f" * [{letter}]({letter}/index.md)\n")