Managing a CryoSPARC Live Session from the CLI

11. Managing a CryoSPARC Live Session from the CLI#

This guide details how to automate the creation, configuration and overall management of a CryoSPARC Live session in Python via the CryoSPARC API.

Note

The commands on this page apply to CryoSPARC v5.

11.1. Setup#

11.1.1. CryoSPARC Connection and Tools API Object#

Specify the master hostname and port number of the CryoSPARC installation where CryoSPARC Live will run.

master_hostname = "localhost"  # should match hostname portion of login token
base_port = 62000  # should match port portion of login token

Connect to the CryoSPARC instance and define the api object.

import time

import cryosparc.tools

cs = cryosparc.tools.CryoSPARC(f"http://{master_hostname}:{base_port}")
assert cs.test_connection()
api = cs.api

11.1.2. Available Parameters#

The full list of session parameters that can be modified can be obtained with the command

api.sessions.get_session_base_params()

11.1.3. Raw Data Specification and Processing Parameters#

If one uses an existing CryoSPARC project, one does not need to define the project_parent_dir, project_title and project_description variables.

project_parent_dir = "/bulk2/CS/prod_projects"  # where to create CryoSPARC project dir

project_title = "Live test"

project_description = "Live processing of the EMPIAR-10025 subset"

session_title = "Live test session"

session_description = "realtime processing of EMPIAR-10025 subset"

exposure_group_configs = [
    {
        "file_engine_watch_path_abs": "/bulk5/data/empiar_10025_subset/",
        "file_engine_filter": "14sep05c_00024sq_*.frames.tif",
        "gainref_path": "/bulk5/data/empiar_10025_subset/norm-amibox05-0.mrc",
    },
    {
        "file_engine_watch_path_abs": "/bulk5/data/empiar_10025_subset/",
        "file_engine_filter": "14sep05c_c_00003gr_00014sq_*.frames.tif",
        "gainref_path": "/bulk5/data/empiar_10025_subset/norm-amibox05-0.mrc",
    },
]

session_params = {
    "psize_A": 0.6575,
    "accel_kv": 300,
    "cs_mm": 2.7,
    "total_dose_e_per_A2": 53,
    "blob_diameter_min": 100,
    "blob_diameter_max": 200,
    "box_size_pix": 440,
    "bin_size_pix": 256,
    "output_f16": True,
    "extract_f16": True,
}

class2D_params = {"class2D_K": 20}

# for ab initio 3D reconstruction
abinit_params = {"abinit_K": 1, "abinit_num_particles": 9000}

refine_params = {"refine_symmetry": "D7"}

11.1.4. Compute Resources#

Specify which scheduler lanes on your CryoSPARC instance to use for various processing tasks.

compute_resources = {
    "phase_one_lane": "slurm_4",  # for motion correction thru particle extraction
    "phase_one_gpus": 1,
    "phase_two_lane": "slurm_4",  # streaming 2D classification and 3D refinement
    "phase_two_ssd": False,  # set True for particle caching
    "auxiliary_lane": "slurm_4",  # transient or optional jobs, like 3d ab initio reconstruction
    "auxiliary_ssd": False,
}

11.1.5. Create CryoSPARC Project or Specify Existing Project’s UID#

project_uid = "P291"  # existing project

To use an existing project, keep the project_uid = "P291" line above. To create a new project instead, comment out that line and uncomment the next cell.

# project_uid = cs.api.projects.create(title=project_title,
#                                     description=project_description,
#                                     parent_dir=project_parent_dir).uid

11.2. Create Session and Capture Session ID#

session_uid = api.sessions.create(project_uid, title=session_title, description=session_description).session_uid

11.2.1. Optional Automatic Start and Pause#

CryoSPARC Live can wait until exposures are available on the filesystem before starting the session. This prevents the session from using hardware resources or license tokens until raw movie files are available for processing.

api.sessions.set_session_phase_one_wait_for_exposures(project_uid, session_uid, phase_one_wait_for_exposures=True)

A session can be paused automatically when it has become idle.

api.sessions.configure_auto_pause(project_uid, session_uid, auto_pause="graceful", auto_pause_after_idle_minutes=5)

11.3. Configuration and Start of Session#

api.sessions.update_compute_configuration(project_uid, session_uid, compute_resources)

# one exposure group may already exist in the newly created session
for i in range(len(exposure_group_configs) - len(api.sessions.find_exposure_groups(project_uid, session_uid))):
    api.sessions.create_exposure_group(project_uid, session_uid)

for eg in zip(api.sessions.find_exposure_groups(project_uid, session_uid), exposure_group_configs):
    api.sessions.update_exposure_group(project_uid, session_uid, eg[0].exp_group_id, eg[1])
    api.sessions.finalize_exposure_group(project_uid, session_uid, eg[0].exp_group_id)

api.sessions.update_session_params(project_uid, session_uid, session_params)

api.sessions.start(project_uid, session_uid)

11.4. Start Streaming 2D Classification#

api.sessions.update_phase2_class2D_params(project_uid, session_uid, class2D_params)
api.sessions.setup_phase2_class2D(project_uid, session_uid)
api.sessions.enqueue_phase2_class2D(project_uid, session_uid)

11.5. Select 2D Classes#

while (
    api.sessions.find_one(project_uid, session_uid).phase2_class2D_num_particles_seen
    < abinit_params["abinit_num_particles"]
):
    time.sleep(10)

api.sessions.select_all_class2d_templates(project_uid, session_uid, "select")

In this example, all templates (and associated particles) are selected. In a production scenario, one may select a meaningful subset of templates using one of the select_class2d_templates_* functions.

11.6. Start ab initio 3D Reconstruction#

api.sessions.update_phase2_abinit_params(project_uid, session_uid, abinit_params)
abinit_job_uid = api.sessions.setup_phase2_abinit(project_uid, session_uid).uid
api.sessions.enqueue_phase2_abinit(project_uid, session_uid)
cs.find_job(project_uid, abinit_job_uid).wait_for_status("completed")

and wait for its completion.

11.7. Start Streaming 3D Refinement#

… using the first (0-based index) volume as input. volume_class_0 is a plausible choice if there was a single class (abinit_K: 1) ab initio 3D reconstruction.

api.sessions.select_phase2_abinit_volume(project_uid, session_uid, volume_name="volume_class_0")
api.sessions.update_phase2_refine_params(project_uid, session_uid, refine_params)
refine_job_uid = api.sessions.setup_phase2_refine(project_uid, session_uid).uid
api.sessions.enqueue_phase2_refine(project_uid, session_uid)

11.8. Export Exposures and Particles#

… for downstream processing. In this example, the export is preceded by periodic checks whether the session has been (manually or automatically) paused or completed.

while api.sessions.find_one(project_uid, session_uid).status not in ["paused", "completed"]:
    time.sleep(10)

exposure_export_job_uid = api.sessions.create_and_enqueue_export_exposures(project_uid, session_uid)
particle_export_job_uid = api.sessions.create_and_enqueue_export_particles(project_uid, session_uid)
Contents