refactor: index.html update (create) from template

.github/workflows/build-installer.yml

@@ -29,6 +29,8 @@ env:
     ESP_IDF_VERSION: ${{ inputs.esp_idf_version }}
     ESPRESSIF_IDE_VERSION: ${{ inputs.espressif_ide_version }}
     ONLINE_INSTALLER_VERSION: ${{ inputs.online_installer_version }}
+    # Based on this defined supported IDF versions are created installer buttons in index.html from releases.json
+    SUPPORTED_IDF_VERSIONS: ('5.2', '4.4', '5.1', '5.0')
 
 jobs:
     build-installer-online:
@@ -57,9 +59,9 @@ jobs:
         secrets: inherit
 
     update-docs-files:
-        needs: build-installer-ide
+        needs: [build-installer-online, build-installer-offline, build-installer-ide]
         name: Create ${{ inputs.installer_type}} installer release PR
-        if: ${{ needs.build-installer-ide.result }} == 'success' || ${{ needs.build-installer-offline.result }} == 'success' || ${{ needs.build-installer-online.result }} == 'success'
+        if: ${{ always() }} && (${{ needs.build-installer-ide.result }} == 'success' || ${{ needs.build-installer-offline.result }} == 'success' || ${{ needs.build-installer-online.result }} == 'success')
         runs-on: windows-latest
         strategy:
             fail-fast: false
@@ -78,6 +80,9 @@ jobs:
                 name: installer-size
                 path: ./
 
+            - name: Install script requirements
+              run: pip install -r scripts/requirements.txt
+
             - name: Get size of online installer and Update docs files
               run: |
                 echo "Instaler size from variable is $(Get-Content variables.txt)"
@@ -100,13 +105,19 @@ jobs:
               run: |
                 Remove-Item -Path variables.txt -Force
 
+            - name: Put current date into a variable
+              run: |
+                $DATE=& Get-Date -format yyyy-MM-dd
+                echo "DATE=$DATE" >> $env:GITHUB_ENV
+
             - name: Create Pull Request
               uses: peter-evans/create-pull-request@v6
               with:
                 token: ${{ secrets.GITHUB_TOKEN }}
                 commit-message: 'Release ${{ env.INSTALLER_TYPE }} installer ${{env.ESPRESSIF_IDE_VERSION}}${{env.ONLINE_INSTALLER_VERSION}} with ESP-IDF v${{ env.ESP_IDF_VERSION }}'
                 title: 'Release ${{ env.INSTALLER_TYPE }} installer ${{env.ESPRESSIF_IDE_VERSION}}${{env.ONLINE_INSTALLER_VERSION}} with ESP-IDF v${{ env.ESP_IDF_VERSION }}'
                 body: '- Updated docs files'
-                branch: 'release-${{ env.INSTALLER_TYPE }}-installer'
+                branch: 'release-${{ env.INSTALLER_TYPE }}${{ env.ESP_IDF_VERSION }}-installer'
                 delete-branch: true
-                base: 'main'
+                base: 'main'
+                reviewers: georgik, jakub-kocka

.github/workflows/build-online-installer.yml

@@ -29,7 +29,7 @@ jobs:
         shell: pwsh
         run: .\Build-Installer.ps1 -InstallerType online
 
-      - name: Create Release
+      - name: Create Release TODO
         id: create_release
         uses: actions/create-release@v1
         env:

scripts/docs_update_release.py

@@ -5,129 +5,97 @@
 
 from datetime import date
 
+from jinja2 import Template, StrictUndefined
+
 # Global constants with paths for files to be changed
 RELEASES_JSON_PATH = "./src/Resources/download/releases.json"
 INDEX_PATH = "./src/Resources/download/index.html"
 INNO_SETUP_PATH = "./src/InnoSetup/IdfToolsSetup.iss"
+INDEX_TEMPLATE_PATH = "./src/Resources/templates/template_index.html"
 
-# Environment variables from GitHub Actions (environmental variables of the runner)
-installer_type: str = environ.get('INSTALLER_TYPE', '')                  # espressif-ide, offline, online
-installer_size: str = argv[1]                                            # e.g. '1.69 GB'
-idf_version = environ.get('ESP_IDF_VERSION', '')                         # e.g. '4.4.7'
-ide_version = environ.get('ESPRESSIF_IDE_VERSION', '')                   # e.g. '2.13.69'
-online_installer_version = environ.get('ONLINE_INSTALLER_VERSION', '')   # e.g. '2.25'
-
-
-def _resolve_installer_type(new_idf_version: str) -> str:
-    """Resolve the type of the installer
-        and return the string of new entry for the index.html
-    """
-    new_entry_ide = f"""            <div class="download-button">
-                    <form method="get" action="https://dl.espressif.com/dl/idf-installer/espressif-ide-setup-{ide_version}-with-esp-idf-{new_idf_version}.exe">
-                        <button class="button-espressif-ide">
-                            <i class="fa fa-download" aria-hidden="true"></i>
-                            <div>Espressif-IDE {ide_version} with ESP-IDF v{new_idf_version} - Offline Installer</div>
-                            <div>Windows 10, 11</div>
-                            <div>Size: {installer_size}</div>
-                        </button>
-                    </form>
-                </div>"""
-
-    new_entry_offline = f"""            <div class="download-button">
-                    <form method="get" action="https://dl.espressif.com/dl/idf-installer/esp-idf-tools-setup-offline-{new_idf_version}.exe">
-                        <button class="button-offline">
-                            <i class="fa fa-download" aria-hidden="true"></i>
-                            <div>ESP-IDF v{new_idf_version} - Offline Installer</div>
-                            <div>Windows 10, 11</div>
-                            <div>Size: {installer_size}</div>
-                        </button>
-                    </form>
-                  </div>"""
-
-    new_entry_online = f"""            <div class="download-button">
-                    <form class="download-form" method="get" action="https://dl.espressif.com/dl/idf-installer/esp-idf-tools-setup-online-{online_installer_version}.exe">
-                        <button class="button-online">
-                            <i class="fa fa-download" aria-hidden="true"></i>
-                            <div>Universal Online Installer {online_installer_version}</div>
-                            <div>Windows 10, 11</div>
-                            <div>Size: {installer_size}</div>
-                        </button>
-                    </form>
-                </div>"""
-
-    """ Installer types acquired from environmental variable of the runner - the same names as with the tags used
-        (No possible way to choose different type - list of options in workflow)
-    """
-    if installer_type == 'espressif-ide':
-        return new_entry_ide
-    elif installer_type == 'offline':
-        return new_entry_offline
-    elif installer_type == 'online':
-        return new_entry_online
-
-
-def update_index(new_idf_version: str):
-    """Update the index.html file with the new release of the installer"""
-    try:
-        with open(path.abspath(INDEX_PATH), "r") as index_file:
-            index_lines = index_file.readlines()
-    except FileNotFoundError as e:
-        raise SystemExit(f"Error opening file {INDEX_PATH} - {e}")
-
-    # find every element with the class "download-button"
-    elements = []
-    for i, line in enumerate(index_lines):
-        if line.strip() == '<div class="download-button">':
-            elements.append([i, index_lines[i:i+10]])   # TODO guess the number dynamically ... regex probably
-            i += 10 # skip the next 10 lines (the length of the element with the class "download-button")
-
-    # choose the elements that contain the installer type
-    selected_elements = []
-    for element in elements:
-        if any(f'{installer_type}' in element_line for element_line in element[1]):
-            selected_elements.append(element)
-
-    print(f"Found {len(selected_elements)} elements with the installer type {installer_type}")
-
+class AddedInstallers:
+    """This class stores if all installer types for all supported versions have been added"""
+    def __init__(self, supported_idf_versions):
+        self.online = False
+        self.offline = False
+        self.espressif_ide = False
 
-    def _replace_installer_button(element_to_replace:list) -> str:
-        """Replace the first occurrence of the installer button with the new one"""
-        element_to_replace = ''.join(element_to_replace[1])
-        print(f"This element will be replaced:\n{element_to_replace}")
+        self.supported_idf_versions = supported_idf_versions
+
+    def all_added(self):
+        """Check if all installer objects for buttons have been added"""
+        return self.online and self.offline and self.espressif_ide and len(self.supported_idf_versions) == 0  
 
-        index_data = ''.join(index_lines)
-        return index_data.replace(element_to_replace, _resolve_installer_type(new_idf_version))
 
-
-    # replace the first occurrence of the offline installer button
-    if installer_type == 'offline':
-        element_to_replace = None
-        for selected_element in selected_elements:
-            for element_line in selected_element[1]:
-                if f'{idf_version[0]}.{idf_version[1]}' in element_line:
-                    element_to_replace = selected_element
+def update_index(releases, supported_idf_versions):
+    """Update the index.html file with the new release of the installer"""
+    added_installers = AddedInstallers(supported_idf_versions)
+
+    online = None
+    espressif_ide = None
+    offline = []
+    for release in releases:
+        if added_installers.all_added():
+            break
+
+        if release['type'] == 'online' and not added_installers.online:
+            online = release
+            added_installers.online = True
+
+        if release['type'] == 'espressif-ide' and not added_installers.espressif_ide:
+            espressif_ide = release
+            added_installers.espressif_ide = True
+
+        if release['type'] == 'offline' and not added_installers.offline:
+            for version in added_installers.supported_idf_versions:
+                if version in release['version']:
+                    offline.append(release)
+                    added_installers.supported_idf_versions.remove(version)
                     break
-        if element_to_replace:
-            new_index_data = _replace_installer_button(element_to_replace)
-        else:   # add new installer button to the top of the offline installer buttons
-            first_occurrence = selected_elements[0][0]
+            if len(added_installers.supported_idf_versions) == 0:
+                added_installers.offline = True
 
-            print(f"First occurrence on line {first_occurrence} - adding new installer button here")
-            index_data = index_lines[0:first_occurrence-1] + list(_resolve_installer_type(new_idf_version)+'\n') + index_lines[first_occurrence:]
-            new_index_data = ''.join(index_data)
-    else:   # replace the first occurrence of the other installer type button
-        new_index_data = _replace_installer_button(selected_elements[0])
+    # sort for offline installer objects for buttons
+    offline_sorted = sorted(offline, key=lambda item: item['version'], reverse=True)
 
+    try:
+        with open(INDEX_TEMPLATE_PATH, 'r') as f:
+            template = f.read()
+    except FileNotFoundError as e:
+        raise SystemExit(f"Error reading file {INDEX_TEMPLATE_PATH} - {e}")
+
+    # StrictUndefined will rise an error if any variable in template is not passed
+    # (instead of silent replace as an empty string)
+    j2_template = Template(template, undefined=StrictUndefined)
+
+    # regex for finding IDE and IDF version in 'version' of espressif_ide object (PATCH part not mandatory)
+    # "2.12.0-with-esp-idf-5.1" -> ['2.12.0', '5.1']
+    pattern = r'\b(\d+\.\d+(?:\.\d+)?)\b'
+    ide_version, ide_idf = re.findall(pattern, espressif_ide['version'])
+
+    # variables to be changed in index.html
+    variables = {
+        "online":{
+            "version": online['version'],
+            "size": online['size'],
+        },
+        "espressif_ide":{
+            "version": ide_version,
+            "idf_version": ide_idf,
+            "size": espressif_ide['size'],
+        },
+        "offline_buttons": offline_sorted
+    }
 
     try:
         with open(path.abspath(INDEX_PATH), "w") as index_file:
-            index_file.write(new_index_data)
+            index_file.write(j2_template.render(variables))
     except FileNotFoundError as e:
         raise SystemExit(f"Error writing file {INDEX_PATH} - {e}")
 
 
 
-def update_releases_json(new_idf_version: str):
+def update_releases_json(new_idf_version: str, installer_type: str, online_installer_version: str, installer_size: str):
     """Update the releases.json file with the new release of the installer"""
     try:
         with open(path.abspath(RELEASES_JSON_PATH), "r") as releases_file:
@@ -152,9 +120,11 @@ def update_releases_json(new_idf_version: str):
             json.dump(releases_json, releases_file, indent=4)
     except FileNotFoundError as e:
         raise SystemExit(f"Error writing file {RELEASES_JSON_PATH} - {e}")
+
+    return releases_json
 
 
-def update_inno_setup():
+def update_inno_setup(installer_type: str, online_installer_version: str, ide_version: str):
     """Update the version of the installer in the InnoSetup file"""
     try:
         with open(path.abspath(INNO_SETUP_PATH), "r") as inno_setup_file:
@@ -179,6 +149,16 @@ def main():
     """Performs the update of all necessary files for the new release of the installer"""   
     if len(argv) < 2:
         raise SystemExit("ERROR: Installer size is not passed as an argument")
+
+    # Environment variables from GitHub Actions (environmental variables of the runner)
+    installer_type: str = environ.get('INSTALLER_TYPE', '')                  # espressif-ide, offline, online
+    installer_size: str = argv[1]                                            # e.g. '1.69 GB'
+    idf_version = environ.get('ESP_IDF_VERSION', '')                         # e.g. '4.4.7'
+    ide_version = environ.get('ESPRESSIF_IDE_VERSION', '')                   # e.g. '2.13.69'
+    online_installer_version = environ.get('ONLINE_INSTALLER_VERSION', '')   # e.g. '2.25'
+
+    supported_idf_versions = eval(environ.get('SUPPORTED_IDF_VERSIONS', "('5.2', '4.4', '5.1', '5.0')"))    # e.g. ('5.2', '4.4', '5.1', '5.0')
+    supported_idf_versions = list(supported_idf_versions)
 
     if not idf_version:
         raise SystemExit("ERROR: IDF version is not provided")
@@ -196,24 +176,24 @@ def main():
     new_idf_version = f"{idf_version[0]}.{idf_version[1]}{f'.{idf_version[2]}' if idf_version[2] else ''}"
 
     if ide_version and not re.match(r'(\d+\.\d+\.\d+)', ide_version):
-        raise SystemExit(f"ERROR: IDE version is not in correct format (it should be 'X.Y.Z')")
+        raise SystemExit(f"ERROR: IDE version is not in correct format (it should be 'X.Y.Z') which '{ide_version}' is not")
 
     if online_installer_version and not re.match(r'(\d+\.\d+)', online_installer_version):
-        raise SystemExit(f"ERROR: Online installer version is not in correct format (it should be 'X.Y')")
+        raise SystemExit(f"ERROR: Online installer version is not in correct format (it should be 'X.Y') which '{online_installer_version}' is not")
 
     if installer_type == 'online' and online_installer_version == '':
         raise SystemExit(f"ERROR: online_installer_version is not provided")
 
     if installer_type == 'espressif-ide' and ide_version == '':
         raise SystemExit(f"ERROR: esp_ide_version or espressif_ide_version is not provided")
 
-    update_releases_json(new_idf_version)
+    releases_json = update_releases_json(new_idf_version, installer_type, online_installer_version, installer_size)
 
     # Update App or IDE version if the installer type is not offline
     if installer_type != 'offline':
-        update_inno_setup()
+        update_inno_setup(installer_type, online_installer_version, ide_version)
 
-    update_index(new_idf_version)
+    update_index(releases_json, supported_idf_versions)
 
     print("Files update done!")
 

scripts/requirements.txt

@@ -0,0 +1,2 @@
+# Template engine for creating index.html in src/Resources/download/index.html from template in scripts/templates/template_index.html
+Jinja2 ~= 3.1