From b627bb89125dae71b1bc0dc6b9503947076b4124 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Mar 2026 13:48:41 +0000 Subject: [PATCH 01/12] Added logic to trigger FIB context and optimised logic for triggering CLEM context --- src/murfey/client/analyser.py | 65 ++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 2d0fe7fd..4ca6afea 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -17,6 +17,7 @@ from murfey.client.context import Context from murfey.client.contexts.atlas import AtlasContext from murfey.client.contexts.clem import CLEMContext +from murfey.client.contexts.fib import FIBContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext from murfey.client.contexts.tomo import TomographyContext @@ -125,20 +126,62 @@ def _find_context(self, file_path: Path) -> bool: """ logger.debug(f"Finding context using file {str(file_path)!r}") + # ----------------------------------------------------------------------------- # CLEM workflow checks - # Look for LIF and XLIF files - if file_path.suffix in (".lif", ".xlif"): + # ----------------------------------------------------------------------------- + if ( + # Look for LIF and XLIF files + file_path.suffix in (".lif", ".xlif") + or ( + # TIFF files have "--Stage", "--Z", and/or "--C" in their file stem + file_path.suffix in (".tiff", ".tif") + and any( + pattern in file_path.stem for pattern in ("--Stage", "--Z", "--C") + ) + ) + ): self._context = CLEMContext("leica", self._basepath, self._token) return True - # Look for TIFF files associated with CLEM workflow - # CLEM TIFF files will have "--Stage", "--Z", and/or "--C" in their file stem - if any( - pattern in file_path.stem for pattern in ("--Stage", "--Z", "--C") - ) and file_path.suffix in (".tiff", ".tif"): - self._context = CLEMContext("leica", self._basepath, self._token) + + # ----------------------------------------------------------------------------- + # FIB workflow checks + # ----------------------------------------------------------------------------- + # Determine if it's from AutoTEM + if ( + # AutoTEM generates a "ProjectData.dat" file + file_path.name == "ProjectData.dat" + or ( + # Images are stored in ".../Sites/Lamella (N)/..." + any(path.startswith("Lamella") for path in file_path.parts) + and "Sites" in file_path.parts + ) + ): + self._context = FIBContext("autotem", self._basepath, self._token) + return True + + # Determine if it's from Maps + if ( + # Electron snapshot metadata in "EMproject.emxml" + file_path.name == "EMproject.emxml" + or ( + # Key images are stored in ".../LayersData/Layer/..." + all(path in file_path.parts for path in ("LayersData", "Layer")) + ) + ): + self._context = FIBContext("maps", self._basepath, self._token) return True + # Determine if it's from Meteor + if ( + # Image metadata stored in "features.json" file + file_path.name == "features.json" or () + ): + self._context = FIBContext("meteor", self._basepath, self._token) + return True + + # ----------------------------------------------------------------------------- # Tomography and SPA workflow checks + # ----------------------------------------------------------------------------- if "atlas" in file_path.parts: self._context = AtlasContext( "serialem" if self._serialem else "epu", self._basepath, self._token @@ -321,6 +364,12 @@ def _analyse(self): ) self.post_transfer(transferred_file) + elif isinstance(self._context, FIBContext): + logger.debug( + f"File {transferred_file.name!r} will be processed as part of the FIB workflow" + ) + self.post_transfer(transferred_file) + elif isinstance(self._context, AtlasContext): logger.debug(f"File {transferred_file.name!r} is part of the atlas") self.post_transfer(transferred_file) From 99f8a2a43ca46a9664eff7532b312b23ab559ca3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Mar 2026 13:49:14 +0000 Subject: [PATCH 02/12] Added placeholder 'elif' blocks for the Maps and Meteor FIB datasets --- src/murfey/client/contexts/fib.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 892a32fb..0b91f7f9 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -45,6 +45,9 @@ def post_transfer( **kwargs, ): super().post_transfer(transferred_file, environment=environment, **kwargs) + # ----------------------------------------------------------------------------- + # AutoTEM + # ----------------------------------------------------------------------------- if self._acquisition_software == "autotem": parts = transferred_file.parts if "DCImages" in parts and transferred_file.suffix == ".png": @@ -123,3 +126,13 @@ def post_transfer( self._lamellae[number]._replace( angle=float(milling_angle.split(" ")[0]) ) + # ----------------------------------------------------------------------------- + # Maps + # ----------------------------------------------------------------------------- + elif self._acquisition_software == "maps": + pass + # ----------------------------------------------------------------------------- + # Meteor + # ----------------------------------------------------------------------------- + elif self._acquisition_software == "meteor": + pass From 6efb99b7ab13672ad103466164f0afca09b45446 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Mar 2026 13:49:38 +0000 Subject: [PATCH 03/12] Added example FIB files to the Analyser test --- tests/client/test_analyser.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 07f8cbb2..0a96bb6e 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -5,6 +5,7 @@ from murfey.client.analyser import Analyser from murfey.client.contexts.atlas import AtlasContext from murfey.client.contexts.clem import CLEMContext +from murfey.client.contexts.fib import FIBContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext from murfey.client.contexts.tomo import TomographyContext @@ -76,6 +77,23 @@ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001_Lng_LVCC.xlif", CLEMContext, ], + # FIB Autotem files + ["visit/autotem/visit/ProjectData.dat", FIBContext], + ["visit/autotem/visit/Sites/Lamella/SetupImages/Preparation.tif", FIBContext], + [ + "visit/autotem/visit/Sites/Lamella (2)//DCImages/DCM_2026-03-09-23-45-40.926/2026-03-09-23-48-43-Finer-Milling-dc_rescan-image-.png", + FIBContext, + ], + # FIB Maps files + ["visit/maps/visit/EMproject.emxml", FIBContext], + [ + "visit/maps/visit/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", + FIBContext, + ], + [ + "visit/maps/visit/LayersData/Layer/Electron Snapshot (2)/Electron Snapshot (2).tiff", + FIBContext, + ], ] From 61d48e5594ebf181bcaf7e88ccae2556d32bd427 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 05:56:52 +0000 Subject: [PATCH 04/12] Added logic to parse FIB Maps metadata and images and trigger workflow (to be determine) once both are accounted for --- src/murfey/client/contexts/fib.py | 129 +++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 0b91f7f9..91582eab 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -1,9 +1,11 @@ from __future__ import annotations import logging +import threading from datetime import datetime from pathlib import Path from typing import Dict, List, NamedTuple, Optional +from xml.etree import ElementTree as ET import xmltodict @@ -13,6 +15,8 @@ logger = logging.getLogger("murfey.client.contexts.fib") +lock = threading.Lock() + class Lamella(NamedTuple): name: str @@ -25,18 +29,98 @@ class MillingProgress(NamedTuple): timestamp: float +class ElectronSnapshotMetadata(NamedTuple): + image_dir: str # Partial path from EMproject.emxml parent to the image + status: str + x_len: float | None + y_len: float | None + z_len: float | None + x_center: float | None + y_center: float | None + z_center: float | None + extent: tuple[float, float, float, float] | None + rotation_angle: float | None + + def _number_from_name(name: str) -> int: return int( name.strip().replace("Lamella", "").replace("(", "").replace(")", "") or 1 ) +def _parse_electron_snapshot_metadata(xml_file: Path): + metadata_dict = {} + root = ET.parse(xml_file).getroot() + datasets = root.findall(".//Datasets/Dataset") + for dataset in datasets: + # Extract all string-based values + name, image_dir, status = [ + node.text + if ((node := dataset.find(node_path)) is not None and node.text is not None) + else "" + for node_path in ( + ".//Name", + ".//FinalImages", + ".//Status", + ) + ] + + # Extract all float values + cx, cy, cz, x_len, y_len, z_len, rotation_angle = [ + float(node.text) + if ((node := dataset.find(node_path)) is not None and node.text is not None) + else None + for node_path in ( + ".//BoxCenter/CenterX", + ".//BoxCenter/CenterY", + ".//BoxCenter/CenterZ", + ".//BoxSize/SizeX", + ".//BoxSize/SizeY", + ".//BoxSize/SizeZ", + ".//RotationAngle", + ) + ] + + # Calculate the extent of the image + extent = None + if ( + cx is not None + and cy is not None + and x_len is not None + and y_len is not None + ): + extent = ( + x_len - (cx / 2), + x_len + (cx / 2), + y_len - (cy / 2), + y_len - (cy / 2), + ) + + # Append metadata for current site to dict + metadata_dict[name] = ElectronSnapshotMetadata( + status=status, + image_dir=image_dir, + x_len=x_len, + y_len=y_len, + z_len=z_len, + x_center=cx, + y_center=cy, + z_center=cz, + extent=extent, + rotation_angle=rotation_angle, + ) + return metadata_dict + + class FIBContext(Context): def __init__(self, acquisition_software: str, basepath: Path, token: str): super().__init__("FIB", acquisition_software, token) self._basepath = basepath self._milling: Dict[int, List[MillingProgress]] = {} self._lamellae: Dict[int, Lamella] = {} + self._electron_snapshots: Dict[str, Path] = {} + self._electron_snapshot_metadata: Dict[str, ElectronSnapshotMetadata] = {} + self._electron_snapshots_submitted: set[str] = set() def post_transfer( self, @@ -130,7 +214,50 @@ def post_transfer( # Maps # ----------------------------------------------------------------------------- elif self._acquisition_software == "maps": - pass + # Electron snapshot metadata file + if transferred_file.name == "EMproject.emxml": + # Extract all "Electron Snapshot" metadata and store it + self._electron_snapshot_metadata = _parse_electron_snapshot_metadata( + transferred_file + ) + # If dataset hasn't been transferred, register it + for dataset_name in list(self._electron_snapshot_metadata.keys()): + if dataset_name not in self._electron_snapshots_submitted: + if dataset_name in self._electron_snapshots: + logger.info(f"Registering {dataset_name!r}") + + ## Workflow to trigger goes here + + # Clear old entry after triggering workflow + self._electron_snapshots_submitted.add(dataset_name) + with lock: + self._electron_snapshots.pop(dataset_name, None) + self._electron_snapshot_metadata.pop(dataset_name, None) + else: + logger.debug(f"Waiting for image for {dataset_name}") + # Electron snapshot image + elif ( + "Electron Snapshot" in transferred_file.name + and transferred_file.suffix in (".tif", ".tiff") + ): + # Store file in Context memory + dataset_name = transferred_file.stem + self._electron_snapshots[dataset_name] = transferred_file + + if dataset_name not in self._electron_snapshots_submitted: + # If the metadata and image are both present, register dataset + if dataset_name in list(self._electron_snapshot_metadata.keys()): + logger.info(f"Registering {dataset_name!r}") + + ## Workflow to trigger goes here + + # Clear old entry after triggering workflow + self._electron_snapshots_submitted.add(dataset_name) + with lock: + self._electron_snapshots.pop(dataset_name, None) + self._electron_snapshot_metadata.pop(dataset_name, None) + else: + logger.debug(f"Waiting for metadata for {dataset_name}") # ----------------------------------------------------------------------------- # Meteor # ----------------------------------------------------------------------------- From cdfd352fa38f14f38ea432a701109551a349edf0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 05:57:28 +0000 Subject: [PATCH 05/12] Added tests and placeholder tests for the FIBContext class --- tests/client/contexts/test_fib.py | 249 ++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 tests/client/contexts/test_fib.py diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py new file mode 100644 index 00000000..f1254296 --- /dev/null +++ b/tests/client/contexts/test_fib.py @@ -0,0 +1,249 @@ +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any + +import pytest + +from murfey.client.contexts.fib import FIBContext + +# ------------------------------------------------------------------------------------- +# FIBContext test utilty functions and fixtures +# ------------------------------------------------------------------------------------- + + +def create_fib_maps_dataset_element( + id: int, + name: str, + relative_path: str, + center_x: float, + center_y: float, + center_z: float, + size_x: float, + size_y: float, + size_z: float, + rotation_angle: float, +): + # Create dataset node + dataset = ET.Element("Dataset") + # ID node + id_node = ET.Element("Id") + id_node.text = str(id) + dataset.append(id_node) + + # Name node + name_node = ET.Element("Name") + name_node.text = name + dataset.append(name_node) + + # Stage position node + box_center = ET.Element("BoxCenter") + for tag, value in ( + ("CenterX", center_x), + ("CenterY", center_y), + ("CenterZ", center_z), + ): + node = ET.Element(tag) + node.text = str(value) + box_center.append(node) + dataset.append(box_center) + + # Image size node + box_size = ET.Element("BoxSize") + for tag, value in ( + ("SizeX", size_x), + ("SizeY", size_y), + ("SizeZ", size_z), + ): + node = ET.Element(tag) + node.text = str(value) + box_size.append(node) + dataset.append(box_size) + + # Rotation angle + angle_node = ET.Element("RotationAngle") + angle_node.text = str(rotation_angle) + dataset.append(angle_node) + + # Relative path + image_path_node = ET.Element("FinalImages") + image_path_node.text = relative_path.replace("/", "\\") + dataset.append(image_path_node) + + return dataset + + +def create_fib_maps_xml_metadata( + project_name: str, + datasets: list[dict[str, Any]], +): + # Create root node + root = ET.Element("EMProject") + + # Project name node + project_name_node = ET.Element("ProjectName") + project_name_node.text = project_name + root.append(project_name_node) + + # Datasets node + datasets_node = ET.Element("Datasets") + for id, dataset in enumerate(datasets): + datasets_node.append(create_fib_maps_dataset_element(id, **dataset)) + root.append(datasets_node) + + return root + + +fib_maps_test_datasets = [ + { + "name": name, + "relative_path": relative_path, + "center_x": cx, + "center_y": cy, + "center_z": cz, + "size_x": sx, + "size_y": sy, + "size_z": sz, + "rotation_angle": ra, + } + for (name, relative_path, cx, cy, cz, sx, sy, sz, ra) in ( + ( + "Electron Snapshot", + "LayersData/Layer/Electron Snapshot", + -0.002, + -0.004, + 0.00000008, + 0.0036, + 0.0024, + 0.0, + 3.1415926535897931, + ), + ( + "Electron Snapshot (2)", + "LayersData/Layer/Electron Snapshot (2)", + -0.002, + -0.004, + 0.00000008, + 0.0036, + 0.0024, + 0.0, + 3.1415926535897931, + ), + ( + "Electron Snapshot (3)", + "LayersData/Layer/Electron Snapshot (3)", + 0.002, + 0.004, + 0.00000008, + 0.0036, + 0.0024, + 0.0, + 3.1415926535897931, + ), + ( + "Electron Snapshot (4)", + "LayersData/Layer/Electron Snapshot (4)", + 0.002, + 0.004, + 0.00000008, + 0.0036, + 0.0024, + 0.0, + 3.1415926535897931, + ), + ) +] + + +@pytest.fixture +def fib_maps_metadata_file(tmp_path: Path): + metadata = create_fib_maps_xml_metadata( + "test-project", + fib_maps_test_datasets, + ) + tree = ET.ElementTree(metadata) + ET.indent(tree, space=" ") + save_path = tmp_path / "EMproject.emxml" + tree.write(save_path, encoding="utf-8") + return save_path + + +@pytest.fixture +def fib_maps_images(tmp_path: Path): + image_list = [] + for dataset in fib_maps_test_datasets: + name = str(dataset["name"]) + relative_path = str(dataset["relative_path"]) + file = tmp_path / relative_path / f"{name}.tiff" + if not file.exists(): + file.parent.mkdir(parents=True, exist_ok=True) + file.touch() + image_list.append(file) + return image_list + + +# ------------------------------------------------------------------------------------- +# Tests +# ------------------------------------------------------------------------------------- +def test_fib_autotem_context(): + pass + + +@pytest.mark.parametrize("metadata_first", ((False, True))) +def test_fib_maps_context( + tmp_path: Path, + fib_maps_metadata_file: Path, + fib_maps_images: list[Path], + metadata_first: bool, +): + # Initialise the FIBContext + basepath = tmp_path + context = FIBContext( + acquisition_software="maps", + basepath=basepath, + token="", + ) + # Assert that its initial state is correct + assert not context._electron_snapshots + assert not context._electron_snapshot_metadata + assert not context._electron_snapshots_submitted + + if metadata_first: + # Read the metadata first + context.post_transfer(fib_maps_metadata_file) + # Metadata field should now be populated + assert all( + name in context._electron_snapshot_metadata.keys() + for name in [image.stem for image in fib_maps_images] + ) + # Parse the images one-by-one + for image in fib_maps_images: + name = image.stem + context.post_transfer(image) + # Entries should now start being removed from 'metadata' and 'images' fields + assert ( + name not in context._electron_snapshots.keys() + and name not in context._electron_snapshot_metadata.keys() + and name in context._electron_snapshots_submitted + ) + else: + # Read in images first + for image in fib_maps_images: + name = image.stem + context.post_transfer(image) + assert ( + name in context._electron_snapshots.keys() + and name not in context._electron_snapshot_metadata.keys() + and name not in context._electron_snapshots_submitted + ) + # Read in the metadata + context.post_transfer(fib_maps_metadata_file) + assert all( + name in context._electron_snapshots_submitted + and name not in context._electron_snapshots.keys() + and name not in context._electron_snapshot_metadata.keys() + for name in [file.stem for file in fib_maps_images] + ) + + +def test_fib_meteor_context(): + pass From 1490b6149ef428c99406c6ebf98621403e6d5e2b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 06:26:54 +0000 Subject: [PATCH 06/12] Use regex to extract number from FIB file name; add logic to include stage slot number and image number for FIB Maps workflow --- src/murfey/client/contexts/fib.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 91582eab..f34079dd 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import re import threading from datetime import datetime from pathlib import Path @@ -30,6 +31,8 @@ class MillingProgress(NamedTuple): class ElectronSnapshotMetadata(NamedTuple): + slot_num: int | None # Which slot in the FIB-SEM it is from + image_num: int image_dir: str # Partial path from EMproject.emxml parent to the image status: str x_len: float | None @@ -43,8 +46,18 @@ class ElectronSnapshotMetadata(NamedTuple): def _number_from_name(name: str) -> int: - return int( - name.strip().replace("Lamella", "").replace("(", "").replace(")", "") or 1 + """ + In the AutoTEM and Maps workflows for the FIB, the sites and images are + auto-incremented with parenthesised numbers (e.g. "Lamella (2)"), with + the first site/image typically not having a number. + + This function extracts the number from the file name, and returns 1 if + no such number is found. + """ + return ( + int(match.group(1)) + if (match := re.search(r"\(([\d+])\)", name)) is not None + else 1 ) @@ -98,6 +111,8 @@ def _parse_electron_snapshot_metadata(xml_file: Path): # Append metadata for current site to dict metadata_dict[name] = ElectronSnapshotMetadata( + slot_num=None if cx is None else (1 if cx < 0 else 2), + image_num=_number_from_name(name), status=status, image_dir=image_dir, x_len=x_len, From c9294f6c032c0efa3ce2c4dadbbef6e4db8a8690 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 06:27:47 +0000 Subject: [PATCH 07/12] Added metadata informaion about 'Status' to simulated FIB Maps metadata --- tests/client/contexts/test_fib.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index f1254296..5fd7c989 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -22,6 +22,7 @@ def create_fib_maps_dataset_element( size_y: float, size_z: float, rotation_angle: float, + status: str, ): # Create dataset node dataset = ET.Element("Dataset") @@ -69,6 +70,11 @@ def create_fib_maps_dataset_element( image_path_node.text = relative_path.replace("/", "\\") dataset.append(image_path_node) + # Status + status_node = ET.Element("Status") + status_node.text = status + dataset.append(status_node) + return dataset @@ -104,6 +110,7 @@ def create_fib_maps_xml_metadata( "size_y": sy, "size_z": sz, "rotation_angle": ra, + "status": "Finished", } for (name, relative_path, cx, cy, cz, sx, sy, sz, ra) in ( ( From 80c41368a95eee34f6734c733cbf6dc40e07345b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 06:40:25 +0000 Subject: [PATCH 08/12] Updated type hints --- src/murfey/client/contexts/fib.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index f34079dd..02dc1c78 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -5,7 +5,7 @@ import threading from datetime import datetime from pathlib import Path -from typing import Dict, List, NamedTuple, Optional +from typing import NamedTuple from xml.etree import ElementTree as ET import xmltodict @@ -22,7 +22,7 @@ class Lamella(NamedTuple): name: str number: int - angle: Optional[float] = None + angle: float | None = None class MillingProgress(NamedTuple): @@ -131,10 +131,10 @@ class FIBContext(Context): def __init__(self, acquisition_software: str, basepath: Path, token: str): super().__init__("FIB", acquisition_software, token) self._basepath = basepath - self._milling: Dict[int, List[MillingProgress]] = {} - self._lamellae: Dict[int, Lamella] = {} - self._electron_snapshots: Dict[str, Path] = {} - self._electron_snapshot_metadata: Dict[str, ElectronSnapshotMetadata] = {} + self._milling: dict[int, list[MillingProgress]] = {} + self._lamellae: dict[int, Lamella] = {} + self._electron_snapshots: dict[str, Path] = {} + self._electron_snapshot_metadata: dict[str, ElectronSnapshotMetadata] = {} self._electron_snapshots_submitted: set[str] = set() def post_transfer( From 808290aeef43fe00404413963026a3b877c456ec Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 07:22:24 +0000 Subject: [PATCH 09/12] Added logic to determine destination path for files in FIB workflow --- src/murfey/client/contexts/fib.py | 56 +++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 02dc1c78..f2a94aee 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -12,7 +12,7 @@ from murfey.client.context import Context from murfey.client.instance_environment import MurfeyInstanceEnvironment -from murfey.util.client import capture_post +from murfey.util.client import capture_post, get_machine_config_client logger = logging.getLogger("murfey.client.contexts.fib") @@ -61,6 +61,39 @@ def _number_from_name(name: str) -> int: ) +def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: + """ + Returns the Path of the file on the client PC. + """ + for s in environment.sources: + if file_path.is_relative_to(s): + return s + return None + + +def _file_transferred_to( + environment: MurfeyInstanceEnvironment, source: Path, file_path: Path, token: str +) -> Path | None: + """ + Returns the Path of the transferred file on the DLS file system. + """ + machine_config = get_machine_config_client( + str(environment.url.geturl()), + token, + instrument_name=environment.instrument_name, + ) + + # Construct destination path + base_destination = Path(machine_config.get("rsync_basepath", "")) / Path( + environment.default_destinations[source] + ) + # Add visit number to the path if it's not present in default destination + if environment.visit not in environment.default_destinations[source]: + base_destination = base_destination / environment.visit + destination = base_destination / file_path.relative_to(source) + return destination + + def _parse_electron_snapshot_metadata(xml_file: Path): metadata_dict = {} root = ET.parse(xml_file).getroot() @@ -144,6 +177,10 @@ def post_transfer( **kwargs, ): super().post_transfer(transferred_file, environment=environment, **kwargs) + if environment is None: + logger.warning("No environment passed in") + return + # ----------------------------------------------------------------------------- # AutoTEM # ----------------------------------------------------------------------------- @@ -257,7 +294,22 @@ def post_transfer( ): # Store file in Context memory dataset_name = transferred_file.stem - self._electron_snapshots[dataset_name] = transferred_file + if not (source := _get_source(transferred_file, environment)): + logger.warning(f"No source found for file {transferred_file}") + return + if not ( + destination_file := _file_transferred_to( + environment=environment, + source=source, + file_path=transferred_file, + token=self._token, + ) + ): + logger.warning( + f"File {transferred_file.name!r} not found on storage system" + ) + return + self._electron_snapshots[dataset_name] = destination_file if dataset_name not in self._electron_snapshots_submitted: # If the metadata and image are both present, register dataset From 8265d804b3bca8d4af1c1080c155c4c77334bc5d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 07:22:44 +0000 Subject: [PATCH 10/12] Updated tests --- tests/client/contexts/test_fib.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index 5fd7c989..e2e7ba9d 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -1,8 +1,10 @@ import xml.etree.ElementTree as ET from pathlib import Path from typing import Any +from unittest.mock import MagicMock import pytest +from pytest_mock import MockerFixture from murfey.client.contexts.fib import FIBContext @@ -197,11 +199,19 @@ def test_fib_autotem_context(): @pytest.mark.parametrize("metadata_first", ((False, True))) def test_fib_maps_context( + mocker: MockerFixture, tmp_path: Path, fib_maps_metadata_file: Path, fib_maps_images: list[Path], metadata_first: bool, ): + # Mock out irrelevant functions + mocker.patch("murfey.client.contexts.fib._get_source", return_value=tmp_path) + mocker.patch( + "murfey.client.contexts.fib._file_transferred_to", side_effect=fib_maps_images + ) + mock_environment = MagicMock() + # Initialise the FIBContext basepath = tmp_path context = FIBContext( @@ -216,7 +226,7 @@ def test_fib_maps_context( if metadata_first: # Read the metadata first - context.post_transfer(fib_maps_metadata_file) + context.post_transfer(fib_maps_metadata_file, mock_environment) # Metadata field should now be populated assert all( name in context._electron_snapshot_metadata.keys() @@ -225,7 +235,7 @@ def test_fib_maps_context( # Parse the images one-by-one for image in fib_maps_images: name = image.stem - context.post_transfer(image) + context.post_transfer(image, mock_environment) # Entries should now start being removed from 'metadata' and 'images' fields assert ( name not in context._electron_snapshots.keys() @@ -236,14 +246,14 @@ def test_fib_maps_context( # Read in images first for image in fib_maps_images: name = image.stem - context.post_transfer(image) + context.post_transfer(image, mock_environment) assert ( name in context._electron_snapshots.keys() and name not in context._electron_snapshot_metadata.keys() and name not in context._electron_snapshots_submitted ) # Read in the metadata - context.post_transfer(fib_maps_metadata_file) + context.post_transfer(fib_maps_metadata_file, mock_environment) assert all( name in context._electron_snapshots_submitted and name not in context._electron_snapshots.keys() From 825f1a4b741926f094f0962e4fc56f4178cb9b4e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 08:14:10 +0000 Subject: [PATCH 11/12] Added placeholder tests for other FIBContext helper functions, added test for '_number_from_name', and fixed broken regex search --- src/murfey/client/contexts/fib.py | 2 +- tests/client/contexts/test_fib.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index f2a94aee..af9e72b0 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -56,7 +56,7 @@ def _number_from_name(name: str) -> int: """ return ( int(match.group(1)) - if (match := re.search(r"\(([\d+])\)", name)) is not None + if (match := re.search(r"^[\w\s]+\((\d+)\)$", name)) is not None else 1 ) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index e2e7ba9d..488b7c7e 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -6,7 +6,7 @@ import pytest from pytest_mock import MockerFixture -from murfey.client.contexts.fib import FIBContext +from murfey.client.contexts.fib import FIBContext, _number_from_name # ------------------------------------------------------------------------------------- # FIBContext test utilty functions and fixtures @@ -193,6 +193,34 @@ def fib_maps_images(tmp_path: Path): # ------------------------------------------------------------------------------------- # Tests # ------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "test_params", + ( # File name | Expected number + # AutoTEM examples + ("Lamella", 1), + ("Lamella (2)", 2), + ("Lamella (12)", 12), + # Maps examples + ("Electron Snapshot", 1), + ("Electron Snapshot (3)", 3), + ("Electron Snapshot (21)", 21), + ), +) +def test_number_from_name(test_params: tuple[str, int]): + name, number = test_params + assert _number_from_name(name) == number + + +def test_get_source(): + pass + + +def test_file_transferred_to(): + pass + + def test_fib_autotem_context(): pass From 9697027e290f761f0152e60eb2ffcafcdd90ef17 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Mar 2026 08:30:56 +0000 Subject: [PATCH 12/12] Added unit test for '_get_source' helper function --- tests/client/contexts/test_fib.py | 34 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index 488b7c7e..e6267b14 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -6,7 +6,7 @@ import pytest from pytest_mock import MockerFixture -from murfey.client.contexts.fib import FIBContext, _number_from_name +from murfey.client.contexts.fib import FIBContext, _get_source, _number_from_name # ------------------------------------------------------------------------------------- # FIBContext test utilty functions and fixtures @@ -164,25 +164,32 @@ def create_fib_maps_xml_metadata( @pytest.fixture -def fib_maps_metadata_file(tmp_path: Path): +def visit_dir(tmp_path: Path): + return tmp_path / "visit" + + +@pytest.fixture +def fib_maps_metadata_file(visit_dir: Path): metadata = create_fib_maps_xml_metadata( "test-project", fib_maps_test_datasets, ) tree = ET.ElementTree(metadata) ET.indent(tree, space=" ") - save_path = tmp_path / "EMproject.emxml" + save_path = visit_dir / "maps/visit/EMproject.emxml" + if not save_path.parent.exists(): + save_path.parent.mkdir(parents=True, exist_ok=True) tree.write(save_path, encoding="utf-8") return save_path @pytest.fixture -def fib_maps_images(tmp_path: Path): +def fib_maps_images(fib_maps_metadata_file: Path): image_list = [] for dataset in fib_maps_test_datasets: name = str(dataset["name"]) relative_path = str(dataset["relative_path"]) - file = tmp_path / relative_path / f"{name}.tiff" + file = fib_maps_metadata_file.parent / relative_path / f"{name}.tiff" if not file.exists(): file.parent.mkdir(parents=True, exist_ok=True) file.touch() @@ -213,8 +220,21 @@ def test_number_from_name(test_params: tuple[str, int]): assert _number_from_name(name) == number -def test_get_source(): - pass +def test_get_source( + tmp_path: Path, + visit_dir: Path, + fib_maps_images: list[Path], + fib_maps_metadata_file: Path, +): + # Mock the MurfeyInstanceEnvironment + mock_environment = MagicMock() + mock_environment.sources = [ + visit_dir, + tmp_path / "another_dir", + ] + # Check that the correct source directory is found + for file in [fib_maps_metadata_file, *fib_maps_images]: + assert _get_source(file, mock_environment) == visit_dir def test_file_transferred_to():