diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 684a4371c..dead3451b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: - name: Install Murfey run: | set -eux - pip install --disable-pip-version-check -e "."[cicd,server,developer] + pip install --disable-pip-version-check -e "."[cicd,server,developer,sxt] - uses: shogo82148/actions-setup-mysql@v1 with: diff --git a/pyproject.toml b/pyproject.toml index 0e02281cb..dbb89e666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,9 @@ server = [ smartem = [ "smartem-decisions[backend]", ] +sxt = [ + "txrm2tiff", +] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues" Documentation = "https://github.com/DiamondLightSource/python-murfey" diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 2d0fe7fd7..b66ef73dc 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -19,6 +19,7 @@ from murfey.client.contexts.clem import CLEMContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext +from murfey.client.contexts.sxt import SXTContext from murfey.client.contexts.tomo import TomographyContext from murfey.client.contexts.tomo_metadata import TomographyMetadataContext from murfey.client.destinations import find_longest_data_directory @@ -110,8 +111,12 @@ def _find_extension(self, file_path: Path) -> bool: if subframe_path := mdoc_data_block.get("SubFramePath"): self._extension = Path(subframe_path).suffix return True - # Check for LIF files separately - elif file_path.suffix == ".lif": + # Check for LIF files and TXRM files separately + elif ( + file_path.suffix == ".lif" + or file_path.suffix == ".txrm" + or file_path.suffix == ".xrm" + ): self._extension = file_path.suffix return True return False @@ -138,6 +143,11 @@ def _find_context(self, file_path: Path) -> bool: self._context = CLEMContext("leica", self._basepath, self._token) return True + # SXT workflow checks + if file_path.suffix in (".txrm", ".xrm"): + self._context = SXTContext("zeiss", self._basepath, self._token) + return True + # Tomography and SPA workflow checks if "atlas" in file_path.parts: self._context = AtlasContext( @@ -321,6 +331,10 @@ def _analyse(self): ) self.post_transfer(transferred_file) + elif isinstance(self._context, SXTContext): + logger.debug(f"File {transferred_file.name!r} is an SXT file") + self.post_transfer(transferred_file) + elif isinstance(self._context, AtlasContext): logger.debug(f"File {transferred_file.name!r} is part of the atlas") self.post_transfer(transferred_file) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 3be79cd77..296c10128 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -13,6 +13,44 @@ logger = logging.getLogger("murfey.client.context") +def _file_transferred_to( + environment: MurfeyInstanceEnvironment, source: Path, file_path: Path, token: str +): + machine_config = get_machine_config_client( + str(environment.url.geturl()), + token, + instrument_name=environment.instrument_name, + ) + if environment.visit in environment.default_destinations[source]: + return ( + Path(machine_config.get("rsync_basepath", "")) + / Path(environment.default_destinations[source]) + / file_path.relative_to(source) # need to strip out the rsync_module name + ) + return ( + Path(machine_config.get("rsync_basepath", "")) + / Path(environment.default_destinations[source]) + / environment.visit + / file_path.relative_to(source) + ) + + +def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: + possible_sources = [] + for s in environment.sources: + if file_path.is_relative_to(s): + possible_sources.append(s) + if not possible_sources: + return None + elif len(possible_sources) == 1: + return possible_sources[0] + source = possible_sources[0] + for extra_source in possible_sources[1:]: + if extra_source.is_relative_to(source): + source = extra_source + return source + + def _atlas_destination( environment: MurfeyInstanceEnvironment, source: Path, token: str ) -> Path: @@ -61,6 +99,10 @@ def ensure_dcg_exists( ) except Exception as e: logger.warning(f"Get EPU session hook failed: {e}") + elif collection_type == "sxt": + experiment_type_id = 47 + session_file = metadata_source / "Session.dm" + source_visit_dir = metadata_source.parent else: logger.error(f"Unknown collection type {collection_type}") return None diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index b67dcac99..86426b385 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -4,8 +4,7 @@ import xmltodict -from murfey.client.context import Context, _atlas_destination -from murfey.client.contexts.spa import _get_source +from murfey.client.context import Context, _atlas_destination, _get_source from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 3b2922f75..94c71ce79 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -7,7 +7,12 @@ import xmltodict -from murfey.client.context import Context, ProcessingParameter +from murfey.client.context import ( + Context, + ProcessingParameter, + _file_transferred_to, + _get_source, +) from murfey.client.destinations import find_longest_data_directory from murfey.client.instance_environment import ( MovieTracker, @@ -26,28 +31,6 @@ logger = logging.getLogger("murfey.client.contexts.spa") -def _file_transferred_to( - environment: MurfeyInstanceEnvironment, source: Path, file_path: Path, token: str -): - machine_config = get_machine_config_client( - str(environment.url.geturl()), - token, - instrument_name=environment.instrument_name, - ) - if environment.visit in environment.default_destinations[source]: - return ( - Path(machine_config.get("rsync_basepath", "")) - / Path(environment.default_destinations[source]) - / file_path.relative_to(source) # need to strip out the rsync_module name - ) - return ( - Path(machine_config.get("rsync_basepath", "")) - / Path(environment.default_destinations[source]) - / environment.visit - / file_path.relative_to(source) - ) - - def _grid_square_metadata_file( f: Path, data_directories: list[Path], visit: str, grid_square: int ) -> Path: @@ -66,22 +49,6 @@ def _grid_square_metadata_file( return metadata_file -def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: - possible_sources = [] - for s in environment.sources: - if file_path.is_relative_to(s): - possible_sources.append(s) - if not possible_sources: - return None - elif len(possible_sources) == 1: - return possible_sources[0] - source = possible_sources[0] - for extra_source in possible_sources[1:]: - if extra_source.is_relative_to(source): - source = extra_source - return source - - def _get_xml_list_index(key: str, xml_list: list) -> int: for i, elem in enumerate(xml_list): if elem["a:Key"] == key: diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 1f617a9eb..2cbe95954 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -4,8 +4,12 @@ import xmltodict -from murfey.client.context import Context, ensure_dcg_exists -from murfey.client.contexts.spa import _file_transferred_to, _get_source +from murfey.client.context import ( + Context, + _file_transferred_to, + _get_source, + ensure_dcg_exists, +) from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post from murfey.util.spa_metadata import ( diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py new file mode 100644 index 000000000..63b0f5832 --- /dev/null +++ b/src/murfey/client/contexts/sxt.py @@ -0,0 +1,224 @@ +import logging +from pathlib import Path +from typing import Any + +from txrm2tiff.inspector import Inspector +from txrm2tiff.txrm import open_txrm +from txrm2tiff.txrm_functions.general import read_stream +from txrm2tiff.xradia_properties.enums import XrmDataTypes + +from murfey.client.context import ( + Context, + _file_transferred_to, + _get_source, + ensure_dcg_exists, +) +from murfey.client.instance_environment import MurfeyInstanceEnvironment +from murfey.util.client import capture_post +from murfey.util.tomo import midpoint + +logger = logging.getLogger("murfey.client.contexts.sxt") + + +class SXTContext(Context): + def __init__(self, acquisition_software: str, basepath: Path, token: str): + super().__init__("SXT", acquisition_software, token) + self._basepath = basepath + + def register_sxt_data_collection( + self, + tilt_series: str, + data_collection_parameters: dict, + file_extension: str, + image_directory: str | Path, + environment: MurfeyInstanceEnvironment | None = None, + ): + if not environment: + logger.error( + "No environment passed to register tomography data collections" + ) + return + try: + metadata_source = ( + self._basepath.parent / environment.visit / self._basepath.name + ) + ensure_dcg_exists( + collection_type="sxt", + metadata_source=metadata_source, + environment=environment, + token=self._token, + ) + + dc_data: dict[str, Any] = { + "experiment_type": "sxt", + "file_extension": file_extension, + "acquisition_software": self._acquisition_software, + "image_directory": str(image_directory), + "data_collection_tag": tilt_series, + "source": str(self._basepath), + "tag": tilt_series, + "pixel_size_on_image": str( + data_collection_parameters.get("pixel_size", 100) + ), + "image_size_x": data_collection_parameters.get("image_size_x", 0), + "image_size_y": data_collection_parameters.get("image_size_y", 0), + "magnification": data_collection_parameters.get("magnification", 0), + "voltage": 0, + } + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow.router", + function_name="start_dc", + token=self._token, + visit_name=environment.visit, + session_id=environment.murfey_session, + data=dc_data, + ) + + recipes_to_assign_pjids = [ + "sxt-tomo-align", + ] + for recipe in recipes_to_assign_pjids: + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow.router", + function_name="register_proc", + token=self._token, + visit_name=environment.visit, + session_id=environment.murfey_session, + data={ + "tag": tilt_series, + "source": str(self._basepath), + "recipe": recipe, + "experiment_type": "sxt", + }, + ) + except Exception as e: + logger.error(f"ERROR {e}, {data_collection_parameters}", exc_info=True) + + def post_transfer( + self, + transferred_file: Path, + environment: MurfeyInstanceEnvironment | None = None, + **kwargs, + ) -> bool: + super().post_transfer( + transferred_file=transferred_file, + environment=environment, + **kwargs, + ) + + data_suffixes = [".txrm"] + + if transferred_file.suffix in data_suffixes and environment: + source = _get_source(transferred_file, environment) + if not source: + logger.warning(f"No source found for file {transferred_file}") + return False + + # Read the tilt angles and pixel size from the txrm + metadata = { + "source": str(self._basepath), + "tilt_series_tag": transferred_file.stem, + } + with open_txrm( + transferred_file, load_images=False, load_reference=False, strict=False + ) as txrm: + inspector = Inspector(txrm) + angles = read_stream( + inspector.txrm.ole, + "ImageInfo/Angles", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + if angles: + metadata["minimum_angle"] = min(angles) + metadata["maximum_angle"] = max(angles) + + pixel_size_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/PixelSize", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + if pixel_size_txrm: + metadata["pixel_size"] = pixel_size_txrm[0] * 1e4 + + image_width_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/ImageWidth", + XrmDataTypes.XRM_INT, + strict=True, + ) + if image_width_txrm: + metadata["image_size_x"] = image_width_txrm[0] + + image_height_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/ImageHeight", + XrmDataTypes.XRM_INT, + strict=True, + ) + if image_height_txrm: + metadata["image_size_y"] = image_height_txrm[0] + + exposure_time_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/ExpTimes", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + if exposure_time_txrm: + metadata["exposure_time"] = exposure_time_txrm[0] + + magnification_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/XrayMagnification", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + if magnification_txrm: + metadata["magnification"] = magnification_txrm[0] + + tilt_count_txrm = read_stream( + inspector.txrm.ole, + "ImageInfo/ImagesTaken", + XrmDataTypes.XRM_INT, + strict=True, + ) + if tilt_count_txrm: + metadata["tilt_count"] = tilt_count_txrm[0] + + self.register_sxt_data_collection( + tilt_series=transferred_file.stem, + data_collection_parameters=metadata, + file_extension=transferred_file.suffix, + image_directory=environment.default_destinations.get( + transferred_file.parent, transferred_file.parent + ), + environment=environment, + ) + + logger.info( + f"The following tilt series will be processed: {transferred_file.stem}" + ) + file_transferred_to = _file_transferred_to( + environment, source, transferred_file, self._token + ) + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow.sxt_router", + function_name="process_sxt_tilt_series", + token=self._token, + visit_name=environment.visit, + session_id=environment.murfey_session, + data={ + "session_id": environment.murfey_session, + "tag": transferred_file.stem, + "source": str(transferred_file.parent), + "pixel_size": metadata.get("pixel_size", 100), + "tilt_offset": midpoint(angles), + "txrm": str(file_transferred_to), + }, + ) + return True diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 155f09370..e0d85d169 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -6,8 +6,12 @@ import xmltodict -from murfey.client.context import Context, ensure_dcg_exists -from murfey.client.contexts.spa import _file_transferred_to, _get_source +from murfey.client.context import ( + Context, + _file_transferred_to, + _get_source, + ensure_dcg_exists, +) from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 54ff1c62c..01712e5f0 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -70,6 +70,10 @@ motion_corrected_mrc, ) from murfey.util.tomo import midpoint +from murfey.workflows.sxt.process_sxt_tilt_series import ( + SXTTiltSeriesInfo, + process_sxt_tilt_series_workflow, +) from murfey.workflows.tomo.tomo_metadata import register_search_map_in_database logger = getLogger("murfey.server.api.workflow") @@ -961,6 +965,25 @@ def _add_tilt(): _add_tilt() +sxt_router = APIRouter( + prefix="/workflow/sxt", + dependencies=[Depends(validate_instrument_token)], + tags=["Workflows: Soft x-ray tomography"], +) + + +@sxt_router.post("/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series") +def process_sxt_tilt_series( + visit_name: str, + session_id: MurfeySessionID, + tilt_series_info: SXTTiltSeriesInfo, + db=murfey_db, +): + return process_sxt_tilt_series_workflow( + visit_name, session_id, tilt_series_info, db + ) + + correlative_router = APIRouter( prefix="/workflow/correlative", dependencies=[Depends(validate_instrument_token)], diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 9858cb800..48d7aa506 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1383,6 +1383,16 @@ murfey.server.api.workflow.spa_router: type: int methods: - POST +murfey.server.api.workflow.sxt_router: + - path: /workflow/sxt/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series + function: process_sxt_tilt_series + path_params: + - name: visit_name + type: str + - name: session_id + type: int + methods: + - POST murfey.server.api.workflow.tomo_router: - path: /workflow/tomo/sessions/{session_id}/tomography_processing_parameters function: register_tomo_proc_params diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py new file mode 100644 index 000000000..53cf7e9c4 --- /dev/null +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -0,0 +1,116 @@ +import logging +from pathlib import Path + +from pydantic import BaseModel +from sqlmodel import select +from werkzeug.utils import secure_filename + +from murfey.server import _transport_object +from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID +from murfey.util import sanitise +from murfey.util.config import get_machine_config +from murfey.util.db import ( + AutoProcProgram, + DataCollection, + DataCollectionGroup, + ProcessingJob, + Session, + TiltSeries, +) + +logger = logging.getLogger("murfey.workflows.sxt.process_sxt_tilt_series") + + +class SXTTiltSeriesInfo(BaseModel): + session_id: int + tag: str + source: str + txrm: str + tilt_series_length: int + pixel_size: float + tilt_offset: int + + +def process_sxt_tilt_series_workflow( + visit_name: str, + session_id: MurfeySessionID, + tilt_series_info: SXTTiltSeriesInfo, + murfey_db: Session, +): + tilt_series_query = murfey_db.exec( + select(TiltSeries) + .where(TiltSeries.session_id == tilt_series_info.session_id) + .where(TiltSeries.tag == tilt_series_info.tag) + .where(TiltSeries.rsync_source == tilt_series_info.source) + ).all() + if tilt_series_query: + tilt_series = tilt_series_query[0] + else: + tilt_series = TiltSeries( + session_id=session_id, + tag=tilt_series_info.tag, + rsync_source=tilt_series_info.source, + tilt_series_length=tilt_series_info.tilt_series_length, + processing_requested=True, + ) + murfey_db.add(tilt_series) + murfey_db.commit() + + collected_ids = murfey_db.exec( + select(DataCollectionGroup, DataCollection, ProcessingJob, AutoProcProgram) + .where(DataCollectionGroup.session_id == session_id) + .where(DataCollectionGroup.tag == tilt_series.rsync_source) + .where(DataCollection.tag == tilt_series.tag) + .where(DataCollection.dcg_id == DataCollectionGroup.id) + .where(ProcessingJob.dc_id == DataCollection.id) + .where(AutoProcProgram.pj_id == ProcessingJob.id) + .where(ProcessingJob.recipe == "sxt-tomo-align") + ).one() + instrument_name = ( + murfey_db.exec(select(Session).where(Session.id == session_id)) + .one() + .instrument_name + ) + machine_config = get_machine_config(instrument_name=instrument_name)[ + instrument_name + ] + + parts = [secure_filename(p) for p in Path(tilt_series_info.txrm).parts] + visit_idx = parts.index(visit_name) + core = Path(*Path(tilt_series_info.txrm).parts[: visit_idx + 1]) + ppath = Path( + "/".join(secure_filename(p) for p in Path(tilt_series_info.txrm).parts) + ) + sub_dataset = "/".join(ppath.relative_to(core).parts[:-1]) + extra_path = machine_config.processed_extra_directory + stack_file = ( + core + / machine_config.processed_directory_name + / sub_dataset + / extra_path + / "Tomograms" + / f"{tilt_series.tag}_stack.mrc" + ) + stack_file.parent.mkdir(parents=True, exist_ok=True) + zocalo_message = { + "recipes": ["sxt-tomo-align"], + "parameters": { + "txrm_file": tilt_series_info.txrm, + "dcid": collected_ids[1].id, + "appid": collected_ids[3].id, + "stack_file": str(stack_file), + "tilt_axis": 0, + "pixel_size": tilt_series_info.pixel_size, + "manual_tilt_offset": -tilt_series_info.tilt_offset, + "node_creator_queue": machine_config.node_creator_queue, + }, + } + if _transport_object: + logger.info( + f"Sending Zocalo message for processing: {sanitise(str(zocalo_message))}" + ) + _transport_object.send("processing_recipe", zocalo_message, new_connection=True) + else: + logger.info( + f"No transport object found. Zocalo message would be {sanitise(str(zocalo_message))}" + ) diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index da22252ab..b22d250cd 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -16,7 +16,7 @@ ) from murfey.util.models import BatchPositionParameters, SearchMapParameters -logger = logging.getLogger("murfey.client.util.tomo_metadata") +logger = logging.getLogger("murfey.workflows.tomo.tomo_metadata") def register_search_map_in_database( diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py new file mode 100644 index 000000000..f309bc461 --- /dev/null +++ b/tests/client/contexts/test_sxt.py @@ -0,0 +1,125 @@ +from unittest.mock import patch +from urllib.parse import urlparse + +from murfey.client.contexts.sxt import SXTContext +from murfey.client.instance_environment import MurfeyInstanceEnvironment + + +def test_sxt_context_initialisation(tmp_path): + context = SXTContext("zeiss", tmp_path, "") + assert context._acquisition_software == "zeiss" + assert context._basepath == tmp_path + + +@patch("requests.post") +def test_sxt_context_xrm(mock_post, tmp_path): + """Currently nothing happens with an xrm file""" + env = MurfeyInstanceEnvironment( + url=urlparse("http://localhost:8000"), + client_id=0, + sources=[tmp_path], + default_destinations={tmp_path: str(tmp_path)}, + instrument_name="", + visit="test", + murfey_session=1, + ) + context = SXTContext("zeiss", tmp_path, "") + return_value = context.post_transfer( + tmp_path / "example.xrm", + required_position_files=[], + required_strings=["fractions"], + environment=env, + ) + assert return_value + mock_post.assert_not_called() + + +@patch("requests.post") +@patch("murfey.client.contexts.sxt.Inspector") +@patch("murfey.client.contexts.sxt.open_txrm") +@patch("murfey.client.contexts.sxt.read_stream") +def test_sxt_context_txrm( + mock_read_stream, mock_open_txrm, mock_inspector, mock_post, tmp_path +): + mock_post().status_code = 200 + mock_read_stream.side_effect = [ + [-55, -25, 5, 35, 65], # Angles + [0.01001], # Pixel size + [1024], # Image Width + [2048], # Image Height + [1.5], # Exposure time + [1000], # Mag + [5], # Image count + ] + + env = MurfeyInstanceEnvironment( + url=urlparse("http://localhost:8000"), + client_id=0, + sources=[tmp_path], + default_destinations={tmp_path: str(tmp_path / "destination")}, + instrument_name="", + visit="test", + murfey_session=1, + ) + context = SXTContext("zeiss", tmp_path, "") + context.post_transfer( + tmp_path / "example.txrm", + required_position_files=[], + required_strings=["fractions"], + environment=env, + ) + + mock_open_txrm.assert_called_once_with( + tmp_path / "example.txrm", load_images=False, load_reference=False, strict=False + ) + mock_inspector.assert_called_once() + + assert mock_post.call_count == 5 + mock_post.assert_any_call( + "http://localhost:8000/workflow/visits/test/sessions/1/register_data_collection_group", + json={ + "experiment_type_id": 47, + "tag": str(tmp_path), + }, + headers={"Authorization": "Bearer "}, + ) + mock_post.assert_any_call( + "http://localhost:8000/workflow/visits/test/sessions/1/start_data_collection", + json={ + "experiment_type": "sxt", + "file_extension": ".txrm", + "acquisition_software": "zeiss", + "image_directory": f"{tmp_path}/destination", + "data_collection_tag": "example", + "source": str(tmp_path), + "tag": "example", + "pixel_size_on_image": "100.1", + "image_size_x": 1024, + "image_size_y": 2048, + "magnification": 1000, + "voltage": 0, + }, + headers={"Authorization": "Bearer "}, + ) + mock_post.assert_any_call( + "http://localhost:8000/workflow/visits/test/sessions/1/register_processing_job", + json={ + "tag": "example", + "source": str(tmp_path), + "recipe": "sxt-tomo-align", + "experiment_type": "sxt", + }, + headers={"Authorization": "Bearer "}, + ) + mock_post.assert_any_call( + "http://localhost:8000/workflow/sxt/visits/test/sessions/1/sxt_tilt_series", + json={ + "session_id": 1, + "tag": "example", + "source": str(tmp_path), + "pixel_size": 100.1, + "tilt_offset": 5, + "txrm": str(tmp_path / "destination/example.txrm"), + }, + headers={"Authorization": "Bearer "}, + ) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 07f8cbb22..b2ae9041e 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -7,6 +7,7 @@ from murfey.client.contexts.clem import CLEMContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext +from murfey.client.contexts.sxt import SXTContext from murfey.client.contexts.tomo import TomographyContext from murfey.client.contexts.tomo_metadata import TomographyMetadataContext from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo @@ -76,6 +77,9 @@ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001_Lng_LVCC.xlif", CLEMContext, ], + # Soft x-ray tomography + ["visit/tomo__tag_ROI10_area1_angle-60to60@1.5_1sec_251p.txrm", SXTContext], + ["visit/X-ray_mosaic_ROI2.xrm", SXTContext], ] diff --git a/tests/workflows/sxt/test_process_sxt_tilt_series.py b/tests/workflows/sxt/test_process_sxt_tilt_series.py new file mode 100644 index 000000000..c14a62200 --- /dev/null +++ b/tests/workflows/sxt/test_process_sxt_tilt_series.py @@ -0,0 +1,106 @@ +from unittest import mock + +from sqlmodel import Session, select + +from murfey.util.db import ( + AutoProcProgram, + DataCollection, + DataCollectionGroup, + ProcessingJob, + TiltSeries, +) +from murfey.workflows.sxt import process_sxt_tilt_series +from tests.conftest import ExampleVisit, get_or_create_db_entry + + +def set_up_db(murfey_db_session: Session): + # Insert common elements needed in all picking tests + dcg_entry: DataCollectionGroup = get_or_create_db_entry( + murfey_db_session, + DataCollectionGroup, + lookup_kwargs={ + "id": 0, + "session_id": ExampleVisit.murfey_session_id, + "tag": "/path/to/tomogram_source", + }, + ) + dc_entry: DataCollection = get_or_create_db_entry( + murfey_db_session, + DataCollection, + lookup_kwargs={ + "id": 0, + "tag": "tomogram_tag", + "dcg_id": dcg_entry.id, + }, + ) + processing_job_entry: ProcessingJob = get_or_create_db_entry( + murfey_db_session, + ProcessingJob, + lookup_kwargs={ + "id": 1, + "recipe": "sxt-tomo-align", + "dc_id": dc_entry.id, + }, + ) + auto_proc_entry = get_or_create_db_entry( + murfey_db_session, + AutoProcProgram, + lookup_kwargs={ + "id": 0, + "pj_id": processing_job_entry.id, + }, + ) + return dcg_entry.id, dc_entry.id, processing_job_entry.id, auto_proc_entry.id + + +@mock.patch("murfey.workflows.sxt.process_sxt_tilt_series._transport_object") +def test_process_new_sxt_tilt_series( + mock_transport, murfey_db_session: Session, tmp_path +): + """Run the picker feedback with less particles than needed for classification""" + dcg_id, dc_id, pj_id, app_id = set_up_db(murfey_db_session) + + new_parameters = process_sxt_tilt_series.SXTTiltSeriesInfo( + session_id=ExampleVisit.murfey_session_id, + tag="tomogram_tag", + source="/path/to/tomogram_source", + txrm=f"{tmp_path}/cm12345-6/raw/tomogram_tag.txrm", + tilt_series_length=5, + pixel_size=100, + tilt_offset=1, + ) + + # Run the registration + process_sxt_tilt_series.process_sxt_tilt_series_workflow( + "cm12345-6", + ExampleVisit.murfey_session_id, + new_parameters, + murfey_db_session, + ) + + # Check the processing message + mock_transport.send.assert_any_call( + "processing_recipe", + { + "parameters": { + "txrm_file": f"{tmp_path}/cm12345-6/raw/tomogram_tag.txrm", + "dcid": dc_id, + "appid": app_id, + "stack_file": f"{tmp_path}/cm12345-6/processed/raw/Tomograms/tomogram_tag_stack.mrc", + "tilt_axis": 0, + "pixel_size": 100, + "manual_tilt_offset": -1, + "node_creator_queue": "node_creator", + }, + "recipes": ["sxt-tomo-align"], + }, + new_connection=True, + ) + + # Check the database insert + tilt_series_entry = murfey_db_session.exec(select(TiltSeries)).one() + assert tilt_series_entry.session_id == ExampleVisit.murfey_session_id + assert tilt_series_entry.tag == "tomogram_tag" + assert tilt_series_entry.rsync_source == "/path/to/tomogram_source" + assert tilt_series_entry.tilt_series_length == 5 + assert tilt_series_entry.processing_requested