Speed up CI: enable micromamba environment caching

.github/workflows/build_conda.yaml

@@ -1,35 +1,35 @@
-
-name: Conda
-
-on:
-  release:
-    types: ['released', 'prereleased']
-
-# workflow_dispatch:        # Un comment line if you also want to trigger action manually
-
-jobs:
-  conda_deployment_with_new_tag:
-    name: Conda deployment of package for platform ${{ matrix.os }} with Python ${{ matrix.python-version }}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - name: Conda environment creation and activation
-        uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830
-        with:
-          python-version: "3.12"
-          environment-file: recipe/build-env.yml
-          activate-environment: build-environment
-          auto-update-conda: true
-          auto-activate-base: false
-          show-channel-urls: true
-      - name: Build and upload the conda packages
-        uses: uibcdf/action-build-and-upload-conda-packages@4940704d2be7906d3bda5b00e3c3e4472fd7808f
-        with:
-          meta_yaml_dir: recipe
-          overwrite: true
-          python-version: "3.12"
-          user: phygbu
-          label: main
-          token: ${{ secrets.ANACONDA }} # Replace with the right name of your secret
+
+name: Conda
+
+on:
+  release:
+    types: ['released', 'prereleased']
+
+# workflow_dispatch:        # Un comment line if you also want to trigger action manually
+
+jobs:
+  conda_deployment_with_new_tag:
+    name: Conda deployment of package for platform ${{ matrix.os }} with Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Conda environment creation and activation
+        uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2.0.7
+        with:
+          environment-file: recipe/build-env.yml
+          environment-name: build-environment
+          create-args: python=3.12
+          init-shell: bash
+      - name: Build and upload the conda packages
+        uses: uibcdf/action-build-and-upload-conda-packages@4940704d2be7906d3bda5b00e3c3e4472fd7808f
+        with:
+          meta_yaml_dir: recipe
+          overwrite: true
+          python-version: "3.12"
+          user: phygbu
+          label: main
+          token: ${{ secrets.ANACONDA }} # Replace with the right name of your secret

.github/workflows/run-tests-action.yaml

@@ -1,92 +1,100 @@
-name: pytest
-on: push
-jobs:
-  run_pytest:
-    name: run-tests (${{ matrix.python-version }}, ${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: bash -l {0}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11", "3.12","3.13"]
-        os: ["ubuntu-latest"]
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-      - name: Install Conda environment with Micromamba
-        uses: conda-incubator/setup-miniconda@v3
-        with:
-          environment-file: tests/test-env.yml
-          python: ${{ matrix.python-version }}
-          channels: conda-forge,phygbu
-          channel-priority: flexible
-          activate-environment: test-environment
-          auto-activate-base: false
-      - name: Conda information
-        run: |
-          conda info
-          conda list
-          conda config --show-sources
-          conda config --show
-      - name: install package
-        run: pip install --no-deps .
-      - name: Install headless server
-        run: |
-           sudo apt-get update
-           sudo apt-get install xvfb
-           sudo apt-get install qtbase5-dev
-      - name: Test with xvfb
-        run: |
-           xvfb-run --auto-servernum /usr/share/miniconda/envs/test-environment/bin/pytest  -n 2 --cov-report= --cov=Stoner --junitxml pytest.xml
-           coverage xml
-        env:
-            TZ: Europe/London
-            LC_CTYPE: en_GB.UTF-8
-            GH_ACTION: True
-      - name: Cleanup X11 server
-        uses: bcomnes/cleanup-xvfb@v1
-      - name: Coveralls Parallel
-        uses: coverallsapp/github-action@v2
-        with:
-          flag-name: run-${{ join(matrix.*, '-') }}
-          format: cobertura
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload Unit Test Results
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: Unit Test Results (Python ${{ matrix.python-version }})
-          path: pytest.xml
-      - name: Post Coveraage result to Codacy
-        run: |
-          export CODACY_PROJECT_TOKEN=${{ secrets.CODACY_PROJECT_TOKEN }}
-          bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml
-
-  publish-test-results:
-    name: "Publish Unit Tests Results"
-    needs: run_pytest
-    runs-on: ubuntu-latest
-    if: always()
-
-    steps:
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-
-      - name: Publish Unit Test Results
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        with:
-          files: artifacts/**/*.xml
-
-  coverage-finish:
-    needs: run_pytest
-    runs-on: ubuntu-latest
-    steps:
-    - name: Coveralls Finished
-      uses: coverallsapp/github-action@v2
-      with:
-        parallel-finished: true
-        carryforward: "run-1,run-2"
+name: pytest
+on: push
+jobs:
+  run_pytest:
+    name: run-tests (${{ matrix.python-version }}, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+      checks: write
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12","3.13"]
+        os: ["ubuntu-latest"]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - name: Install Mamba environment
+        uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2.0.7
+        with:
+          environment-file: tests/test-env.yml
+          environment-name: test-environment
+          create-args: >-
+            python=${{ matrix.python-version }}
+            --channel-priority flexible
+          init-shell: bash
+          cache-downloads: true
+          cache-environment: true
+      - name: Mamba information
+        run: |
+          micromamba info
+          micromamba list
+      - name: install package
+        run: pip install --no-deps .
+      - name: Install headless server
+        run: |
+           sudo apt-get update
+           sudo apt-get install xvfb
+           sudo apt-get install qtbase5-dev
+      - name: Test with xvfb
+        run: |
+           xvfb-run --auto-servernum pytest -n 2 --cov-report= --cov=Stoner --junitxml pytest.xml
+           coverage xml
+        env:
+            TZ: Europe/London
+            LC_CTYPE: en_GB.UTF-8
+            GH_ACTION: True
+      - name: Cleanup X11 server
+        uses: bcomnes/cleanup-xvfb@9e016c43bb8d73fe7d5933d2ef00fd770c1a7c50 # v1.0.9
+      - name: Coveralls Parallel
+        uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
+        with:
+          flag-name: run-${{ join(matrix.*, '-') }}
+          format: cobertura
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: Unit Test Results (Python ${{ matrix.python-version }})
+          path: pytest.xml
+      - name: Post Coveraage result to Codacy
+        run: |
+          export CODACY_PROJECT_TOKEN=${{ secrets.CODACY_PROJECT_TOKEN }}
+          bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml
+
+  publish-test-results:
+    name: "Publish Unit Tests Results"
+    needs: run_pytest
+    runs-on: ubuntu-latest
+    if: always()
+    permissions:
+      checks: write
+      pull-requests: write
+
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          path: artifacts
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@c950f6fb443cb5af20a377fd0dfaa78838901040 # v2.23.0
+        with:
+          files: artifacts/**/*.xml
+
+  coverage-finish:
+    needs: run_pytest
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
+      with:
+        parallel-finished: true
+        carryforward: "run-1,run-2"

README.rst

@@ -199,7 +199,7 @@ making beta packages available.
 Development Version
 -------------------
 
-The current development version is currently on a stable branch and is version 0.11.x. The master branch contains work
+The current development version is currently on a stable branch and is version 0.11.x. The main branch contains work
 in progress to migrate to using Pandas dataframes as the underlying data store - this is however largely broken!
 
 Build Status

Stoner/analysis/functions.py

@@ -175,7 +175,7 @@ def decompose(
         ycol = [ycol]
 
     if hysteretic:
-        from .Util import split_up_down  # pylint: disable=import-outside-toplevel
+        from .utils import split_up_down  # pylint: disable=import-outside-toplevel
 
         fldr = split_up_down(datafile, datafile.xcol)
         for grp in ["rising", "falling"]:

Stoner/core/array.py

@@ -481,7 +481,7 @@ def _col_args(
                 break
         else:  # User didn't set any values, setas will win
             no_guess = kwargs.get("no_guess", False)
-        ret = AttributeStore(self.setas._get_cols(no_guess=no_guess))
+        ret = AttributeStore(self.setas._get_cols(no_guess=no_guess, startx=kwargs.get("startx", 0)))
         force_list = kwargs.get("force_list", not scalar)
         for c in list(cols.keys()):
             if isnone(cols[c]):  # Not defined, fallback on setas

Stoner/formats/data/zip.py

@@ -1,10 +1,15 @@
 # -*- coding: utf-8 -*-
 """Loader for zip files."""
+import fnmatch
+import json
 import pathlib
 import zipfile as zf
 from os import path
 from traceback import format_exc
 
+import chardet
+import pandas as pd
+
 from ...compat import path_types, str2bytes
 from ...core.data import Data
 from ...core.exceptions import StonerLoadError
@@ -14,6 +19,8 @@
 from ..decorators import register_loader, register_saver
 from ..utils.zip import test_is_zip
 
+from ...tools.json import flatten_json, find_paths, find_parent_dicts
+
 
 def _split_filename(filename: Filename, **kwargs: Kwargs) -> Filename:
     """Try to get the member and filename parts."""
@@ -27,7 +34,63 @@ def _split_filename(filename: Filename, **kwargs: Kwargs) -> Filename:
     return filename
 
 
-@register_loader(patterns=(".zip", 16), mime_types=("application/zip", 16), name="ZippedFile", what="Data")
+@register_loader(patterns=(".mlseq", 16), mime_types=("application/zip", 16), name="MeasureLinkFile", what="Data")
+def load_measure_linkfile(new_data: Data, *args: Args, **kwargs: Kwargs) -> Data:
+    """Load a MeasureLink sequence file and assemble as a data object.
+
+    Args:
+        new_data (Data):
+            Data instance into whoch to load the new data.
+        *args:
+            Other positional arguments passed to get_filename.
+
+    Keyword Arguments:
+        **kwargs:
+            Other keyword arguments passed to get_filename.
+
+    Returns:
+        (Data):
+            Loaded Data instance.
+
+    Notes:
+        `.mlseq` files are actually zip archives containing a collection of json files and a flat list of sub-folders
+        The subfolders contain json for the node operations and optionally (if the key HasData is True) a csv file.
+    """
+    filename, args, kwargs = get_filename(args, kwargs)
+    if not test_is_zip(filename):
+        raise StonerLoadError("Must be a zip file to load as a measurement sequence.")
+    with zf.ZipFile(filename, "r") as seq:
+        if "FileInfo.json" not in seq.namelist():
+            raise StonerLoadError("Missing the Measurelink Sequence FileInfo.json entry")
+        with seq.open("FileInfo.json", "r") as fileinfo_json:
+            fileinfo = fileinfo_json.read()
+            fileinfo = fileinfo.decode(chardet.detect(fileinfo)["encoding"])
+        fileinfo = json.loads(fileinfo)
+        new_data.metadata.update(flatten_json(fileinfo))
+        with seq.open("Model.json", "r") as model_json:
+            model = model_json.read()
+            model = model.decode(chardet.detect(model)["encoding"])
+        model = json.loads(model)
+        # new_data.metadata.update(flatten_json(model))
+        for ix, pth in enumerate(fnmatch.filter(seq.namelist(), "*.csv")):
+            with seq.open(pth) as dataframe:
+                df = pd.read_csv(dataframe)
+            if ix == 0:
+                data = df
+            else:
+                data = pd.concat([data, df])
+
+        data = data.select_dtypes(include="number")
+        new_data.data = data.values
+        new_data.column_headers = list(data.columns)
+
+        has_data = find_paths(model, "HasData", True)
+
+    new_data.filename = filename
+    return new_data
+
+
+@register_loader(patterns=(".zip", 24), mime_types=("application/zip", 16), name="ZippedFile", what="Data")
 def load_zipfile(new_data: Data, *args: Args, **kwargs: Kwargs) -> Data:
     """Load a file from the zip file, opening it as necessary.
 

Stoner/formats/utils/zip.py

@@ -23,7 +23,13 @@ def test_is_zip(filename, member=""):
     """
     if not filename or str(filename) == "":
         return False
-    if zf.is_zipfile(filename):
+    if isinstance(filename, (bytes, bytearray)) and b"\x00" in filename:
+        return False
+    try:
+        is_zip = zf.is_zipfile(filename)
+    except (ValueError, TypeError, OSError):
+        return False
+    if is_zip:
         return filename, member
     part = path.basename(filename)
     newfile = path.dirname(filename)