Release workflow update (#202)

* Fix upload paths in release.py

* Add toolbox creation functions

* Copy CITATION and license file for mltbx instead

* Fix build failure

* Release script now errors if no wheels or mltbx exist

Also move eigen folder outside of external/ so it is
not included in the ctf file (bloats it by 20MB)

* Dummy release 4.0.1 to test script

* Bugfixes to release.py

* [no ci] Add developer docs on release workflow

* [no ci] Revert version in CITATION and CHANGELOG

.github/workflows/build_pyspinw.yml

@@ -52,6 +52,38 @@ jobs:
           name: mex-${{ matrix.os }}
           path: ${{ github.workspace }}/external/**/*.mex*
 
+  build_mltbx:
+    runs-on: ubuntu-latest
+    needs: compile_mex
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout SpinW
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Download MEX artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: mex-*
+          path: ${{ github.workspace }}/external
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          release: latest
+      - name: Build mltbx
+        uses: matlab-actions/run-command@v2
+        with:
+          command: "cd mltbx; create_mltbx"
+      - name: Upload mltbx
+        uses: actions/upload-artifact@v4
+        with:
+          name: spinw.mltbx
+          path: ${{ github.workspace }}/mltbx/spinw.mltbx
+      - name: Setup tmate
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+
   build_ctfs:
     needs: compile_mex
     runs-on: self-hosted
@@ -89,6 +121,11 @@ jobs:
         with:
           pattern: ctf-*
           path: python/ctf
+      - name: Download mltbx artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: spinw.mltbx
+          path: mltbx
       - name: Set up Python environment
         uses: actions/setup-python@v4
         with:
@@ -119,7 +156,7 @@ jobs:
       - name: Build Wheel
         run: |
           cd ${{ github.workspace }}/python
-          python -m pip wheel --no-deps --wheel-dir build .
+          python -m pip wheel --no-deps --wheel-dir wheelhouse .
       - name: Run python test
         run: |
           pip install scipy
@@ -135,8 +172,8 @@ jobs:
       - name: Upload release wheels
         if: ${{ github.event_name == 'release' }}
         run: |
-          pip3 install requests
-          python3 release.py --notest --github --token=${{ secrets.GH_TOKEN }}
+          python -m pip install requests
+          python release.py --notest --github --token=${{ secrets.GH_TOKEN }}
       - name: Setup tmate
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3

.github/workflows/publish_pypi.yml

@@ -13,10 +13,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/setup-python@v4
       - name: Download wheels
         run: |
-          python3 -m pip install twine requests
-          python3 release.py --pypi --token=${{ secrets.GH_TOKEN }}
+          python -m pip install twine requests
+          python release.py --pypi --token=${{ secrets.GH_TOKEN }}
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/release.yml

@@ -13,10 +13,12 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - if: |
+      - uses: actions/setup-python@v4
+      - name: Create Release
+        if: |
           contains(github.event.pull_request.title, 'RELEASE') &&
           github.event.pull_request.merged
         shell: bash -l {0}
         run: |
-          pip3 install requests
-          python3 release.py --notest --github --create_tag --token=${{ secrets.GH_TOKEN }}
+          python -m pip install requests
+          python release.py --notest --github --create_tag --token=${{ secrets.GH_TOKEN }}

.gitignore

@@ -38,3 +38,13 @@ unresolvedSymbols.txt
 
 coverage*xml
 junit*xml
+coverageReport
+coverage.html
+
+mltbx/mltbx/CITATION.cff
+mltbx/mltbx/license.txt
+mltbx/spinw.mltbx
+
+*.mexa64
+*.mexmaci64
+*.mexw64

docs/developers.md

@@ -0,0 +1,32 @@
+# Release workflow
+
+The release workflow is mostly automated using continuous integration builds,
+but some actions and triggering a release needs to be done manually by the developer.
+
+To create a release:
+
+1. Create a branch and edit the `CHANGELOG.md` and `CITATION.cff` files to update it with a new version number.
+2. Create a new PR from the branch. The PR must have `RELEASE` in the title.
+3. This will trigger a build with multiple versions of python.
+4. Review the branch, check that all tests for all python versions pass and if so merge.
+5. Once merged, the CI should create a github release in "Draft" mode.
+6. Check that the release page is correct (has the wheel and `mltbx` files and the release notes are ok).
+7. Check that the wheel and `mltbx` toolbox can be installed and work.
+8. Then manually trigger the `Publish to PyPI` action to upload the wheel to PyPI.
+
+
+In particular, in step 1:
+
+* in `CHANGELOG.md` the first title line must have the form:
+
+```
+# [<newver>](https://github.com/spinw/spinw/compare/<oldver>...<newver>)
+```
+
+* in `CITATION.cff` the `version` field must be updated with a new version
+
+If the version string in these two files do not match, or if the version string matches an existing git tag,
+then the CI build will fail.
+
+Also note that in step 8, after uploading to PyPI the release cannot be changed on PyPI (only deleted).
+If a release is deleted, you have to then create a new release version (PyPI does not allow overwriting previous releases).

mltbx/create_mltbx.m

@@ -0,0 +1,13 @@
+currdir = fileparts(mfilename('fullpath'));
+mltbx_dir = fullfile(currdir, 'mltbx');
+if exist(mltbx_dir)
+    rmdir(mltbx_dir, 's');
+end
+mkdir(fullfile(currdir, 'mltbx'));
+copyfile(fullfile(currdir, '..', 'CITATION.cff'), fullfile(currdir, 'mltbx'));
+copyfile(fullfile(currdir, '..', 'license.txt'), fullfile(currdir, 'mltbx'));
+copyfile(fullfile(currdir, '..', 'swfiles'), fullfile(currdir, 'mltbx', 'swfiles'));
+copyfile(fullfile(currdir, '..', 'external'), fullfile(currdir, 'mltbx', 'external'));
+copyfile(fullfile(currdir, '..', 'dat_files'), fullfile(currdir, 'mltbx', 'dat_files'));
+copyfile(fullfile(currdir, 'spinw_on.m'), fullfile(currdir, 'mltbx'));
+matlab.addons.toolbox.packageToolbox("spinw.prj", "spinw.mltbx");

mltbx/spinw.prj

@@ -0,0 +1,111 @@
+<deployment-project plugin="plugin.toolbox" plugin-version="1.0">
+  <configuration file="spinw.prj" location="" name="spinw" target="target.toolbox" target-name="Package Toolbox">
+    <param.appname>SpinW</param.appname>
+    <param.authnamewatermark>Duc Le</param.authnamewatermark>
+    <param.email>duc.le@stfc.ac.uk</param.email>
+    <param.company />
+    <param.summary>A library for spin wave calculation</param.summary>
+    <param.description>A library for spin wave calculation</param.description>
+    <param.screenshot />
+    <param.version>4.0</param.version>
+    <param.output>${PROJECT_ROOT}/spinW.mltbx</param.output>
+    <param.products.name />
+    <param.products.id />
+    <param.products.version />
+    <param.platforms />
+    <param.guid>859ef126-ff2e-4f45-8bff-60126ff3ff61</param.guid>
+    <param.exclude.filters />
+    <param.exclude.pcodedmfiles>true</param.exclude.pcodedmfiles>
+    <param.examples />
+    <param.demosxml />
+    <param.apps />
+    <param.registered.apps />
+    <param.docs />
+    <param.getting.started.guide />
+    <param.matlabpath.excludes />
+    <param.javaclasspath.excludes />
+    <param.exported.on.package>false</param.exported.on.package>
+    <param.required.addons />
+    <param.matlab.project.id />
+    <param.matlab.project.name />
+    <param.release.start />
+    <param.release.end />
+    <param.release.current.only>false</param.release.current.only>
+    <param.compatiblity.windows>true</param.compatiblity.windows>
+    <param.compatiblity.macos>true</param.compatiblity.macos>
+    <param.compatiblity.linux>true</param.compatiblity.linux>
+    <param.compatiblity.matlabonline>true</param.compatiblity.matlabonline>
+    <param.installation.map />
+    <param.additional.sw.names />
+    <param.additional.sw.licenses />
+    <param.additional.sw.win.url />
+    <param.additional.sw.mac.url />
+    <param.additional.sw.linux.url />
+    <unset>
+      <param.company />
+      <param.screenshot />
+      <param.output />
+      <param.products.name />
+      <param.products.id />
+      <param.products.version />
+      <param.platforms />
+      <param.exclude.filters />
+      <param.exclude.pcodedmfiles />
+      <param.examples />
+      <param.demosxml />
+      <param.apps />
+      <param.registered.apps />
+      <param.docs />
+      <param.getting.started.guide />
+      <param.matlabpath.excludes />
+      <param.javaclasspath.excludes />
+      <param.exported.on.package />
+      <param.required.addons />
+      <param.matlab.project.id />
+      <param.matlab.project.name />
+      <param.release.start />
+      <param.release.end />
+      <param.release.current.only />
+      <param.compatiblity.windows />
+      <param.compatiblity.macos />
+      <param.compatiblity.linux />
+      <param.compatiblity.matlabonline />
+      <param.installation.map />
+      <param.additional.sw.names />
+      <param.additional.sw.licenses />
+      <param.additional.sw.win.url />
+      <param.additional.sw.mac.url />
+      <param.additional.sw.linux.url />
+    </unset>
+    <fileset.rootdir>
+      <file>${PROJECT_ROOT}/mltbx</file>
+    </fileset.rootdir>
+    <fileset.rootfiles />
+    <fileset.depfun.included />
+    <fileset.depfun.excluded />
+    <fileset.package />
+    <build-deliverables>
+      <file location="${PROJECT_ROOT}" name="spinW.mltbx" optional="false">spinW.mltbx</file>
+    </build-deliverables>
+    <workflow />
+    <matlab>
+      <root>D:/MATLAB/R2024a</root>
+      <toolboxes />
+    </matlab>
+    <platform>
+      <unix>false</unix>
+      <mac>false</mac>
+      <windows>true</windows>
+      <win2k>false</win2k>
+      <winxp>false</winxp>
+      <vista>false</vista>
+      <linux>false</linux>
+      <solaris>false</solaris>
+      <osver>10.0</osver>
+      <os32>false</os32>
+      <os64>true</os64>
+      <arch>win64</arch>
+      <matlab>true</matlab>
+    </platform>
+  </configuration>
+</deployment-project>

mltbx/spinw_on.m

@@ -0,0 +1,6 @@
+if exist('spinw', 'class') ~= 8
+    currdir = fileparts(mfilename('fullpath'));
+    addpath(genpath(fullfile(currdir, 'swfiles')));
+    addpath(genpath(fullfile(currdir, 'external')));
+    addpath(genpath(fullfile(currdir, 'dat_files')));
+end

release.py

@@ -32,7 +32,7 @@ def main():
 
 
 def release_github(test=True, create_tag=False, token=None):
-    rv = subprocess.run(['git', 'describe', '--tags', '--dirty', '--always', '--long'],
+    rv = subprocess.run(['git', 'describe', '--tags', '--always'],
             capture_output=True)
     if rv.returncode != 0:
         raise Exception(f'During git describe, got this error: {rv.stderr}')
@@ -51,7 +51,7 @@ def release_github(test=True, create_tag=False, token=None):
                      re.DOTALL | re.MULTILINE).groups()[0].strip()
     payload = {
         "tag_name": git_ver,
-        "target_commitish": "main",
+        "target_commitish": "master",
         "name": git_ver,
         "body": desc,
         "draft": False,
@@ -156,16 +156,33 @@ def _create_gh_release(payload, token):
 def _upload_assets(upload_url, token):
     assert token is not None, 'Need token for this action'
     import requests
-    for wheelpath in ['build', 'dist', 'wheelhouse']:
-        wheelfile = os.path.basename(wheelpath)
-        print(f'Uploading wheel {wheelpath}')
-        with open(wheelpath, 'rb') as f:
+    wheelpaths = None
+    wheelhouse = os.path.join('python', 'wheelhouse')
+    if os.path.exists(wheelhouse):
+        wheelpaths = [os.path.join(wheelhouse, ff) for ff in os.listdir(wheelhouse)]
+    if wheelpaths is not None:
+        for wheelpath in wheelpaths:
+            wheelfile = os.path.basename(wheelpath)
+            print(f'Uploading wheel {wheelpath}')
+            with open(wheelpath, 'rb') as f:
+                upload_response = requests.post(
+                    f"{upload_url}?name={wheelfile}",
+                    headers={"Authorization": "token " + token,
+                             "Content-type": "application/octet-stream"},
+                    data=f.read())
+                print(upload_response.text)
+    mltbx = os.path.join('mltbx', 'spinw.mltbx')
+    if os.path.exists(mltbx):
+        print('Uploading mltbx')
+        with open(mltbx, 'rb') as f:
             upload_response = requests.post(
-                f"{upload_url}?name={wheelfile}",
+                f"{upload_url}?name={mltbx}",
                 headers={"Authorization": "token " + token,
                          "Content-type": "application/octet-stream"},
                 data=f.read())
             print(upload_response.text)
+    elif wheelpaths is None:
+        raise RuntimeError('No wheels or matlab-toolboxes found in folder. Cannot upload anything')
     return None
 
 

swfiles/sw_mex.m

@@ -47,8 +47,8 @@ function sw_mex(varargin)
     swloop_dir = [sw_rootdir filesep 'external' filesep 'swloop'];
     swqconv_dir = [sw_rootdir filesep 'external' filesep 'sw_qconv'];
     eigen_ver = '3.4.0';
-    if ~exist([swloop_dir filesep 'eigen-' eigen_ver], 'dir')
-        cd(swloop_dir);
+    if ~exist([sw_rootdir filesep 'eigen-' eigen_ver], 'dir')
+        cd(sw_rootdir);
         disp('Downloading Eigen')
         mkdir(['eigen-' eigen_ver]);
         cd(['eigen-' eigen_ver]);
@@ -70,7 +70,7 @@ function sw_mex(varargin)
         mex('-v','-largeArrayDims','sw_mtimesx.c','-lmwblas','COMPFLAGS=$COMPFLAGS /openmp',...
             'LINKFLAGS=$LINKFLAGS /nodefaultlib:vcomp "$MATLABROOT\bin\win64\libiomp5md.lib"')
         cd(swloop_dir);
-        mex('-R2018a',['COMPFLAGS= /I eigen-' eigen_ver],'swloop.cpp')
+        mex('-R2018a',['COMPFLAGS= /I ' sw_rootdir filesep 'eigen-' eigen_ver],'swloop.cpp')
     elseif ismac
         % add =libiomp5 after -fopenmp?
         if strcmp(mexext, 'mexmaca64')
@@ -98,7 +98,7 @@ function sw_mex(varargin)
             ['LDFLAGS="$LDFLAGS ' mwlinklib ' ' omp_lib ' -lmwblas"'], ...
             'CXXOPTIMFLAGS="$CXXOPTIMFLAGS -Xclang -fopenmp"', omp_inc);
         cd(swloop_dir);
-        mex('-v','-R2018a',['-Ieigen-' eigen_ver],'swloop.cpp')
+        mex('-v','-R2018a',['-I' sw_rootdir filesep 'eigen-' eigen_ver],'swloop.cpp')
     else
         % linux?
         cd(eig_omp_dir);
@@ -108,7 +108,7 @@ function sw_mex(varargin)
         cd(mtimesx_dir);
         mex('-v','-largeArrayDims','sw_mtimesx.c','-lmwblas','CXXFLAGS=$CXXFLAGS -fopenmp -pthread','LDFLAGS=$LDFLAGS -fopenmp')
         cd(swloop_dir);
-        mex('-v','-R2018a',['-Ieigen-' eigen_ver],'swloop.cpp')
+        mex('-v','-R2018a',['-I' sw_rootdir filesep 'eigen-' eigen_ver],'swloop.cpp')
     end
     cd(swqconv_dir);
     mex('-R2018a','sw_qconv.cpp')