@article {Lee2022a, title = {Evaluating Faster R-CNN for cataract surgery tool detection using microscopy video}, year = {2022}, abstract = {

Introduction: Traditional methods of cataract surgery skill assessment rely on human expert supervision. This exposes the trainee to interobserver variability and inconsistent feedback. Alternative measures such as sensorbased instrument motion analysis promise objective assessment [1]. However, sensor-based systems are logistically complicated and expensive to obtain. Previous studies have demonstrated a strong correlation between sensor-based metrics and two-dimensional motion metrics obtained from object detection [2]. Reliable object detection is the foundation for computing such performance metrics. Therefore, the objective of this study is to evaluate the performance of an object detection network, namely Faster Region-Based Convolutional Neural Network (FRCNN), in recognition of cataract surgery tools in microscopy video. Methods: Microscope video was recorded for 25 trials of cataract surgery on an artificial eye. The trials were performed by a cohort consisting of one senior-surgeon and four junior-surgeons and manually annotated for bounding box locations of the cataract surgery tools (Figure 1) The surgical tools used included: forceps, diamond keratomes, viscoelastic cannulas, and cystotome needles. A FRCNN [3] was trained on a total of 130,614 frames for object detection. We used five-fold cross validation, using a leave-one-userout method. In this manner, all videos from one surgeon were reserved for testing and the frames from the remaining 20 videos were divided among training and validation. Network performance was evaluated via mean average precision (mAP), which is defined as the area under the precision/recall curve. Samples were considered correctly identified when the intersection over union (IoU) between the ground truth and predicted bounding boxes was greater than 0.5. Results: The overall mAP of the network was 0.63. Toolspecific mAPs ranged between 0.49 and 0.96 (Table 1). The high accuracy in detection of the cystotome needle is likely due to the distinct size and shape of the tool tip. The diamond keratome had the lowest mAP of any of the tools recognized, however this may be attributed to variations in the appearance of the tool tip (Figure 2). Conclusions: The FRCNN was able to recognize the surgical tools used in cataract surgery with reasonably high accuracy. Now that we know the network can sufficiently recognize the surgical tools, our next goal is to use this network to compute motion-based performance metrics. Future work seeks to validate these performance metrics against those obtained from sensor-based tracking and against expert evaluations. This serves as a first step towards providing consistent and accessible feedback for future trainees learning cataract surgery.\ 

}, author = {H.Y. Lee}, editor = {R. Hisey} } @conference {ODriscoll2022a, title = {Feasibility of object detection for skill assessment in central venous catheterization}, booktitle = {SPIE Medical Imaging}, year = {2022}, month = {02/2022}, publisher = {SPIE Medical Imaging}, organization = {SPIE Medical Imaging}, address = {San Diego}, abstract = {

Purpose: Computer-assisted surgical skill assessment methods have traditionally relied on tracking tool motion with physical sensors. These tracking systems can be expensive, bulky, and impede tool function. Recent advances in object detection networks have made it possible to quantify tool motion using only a camera. These advances open the door for a low-cost alternative to current physical tracking systems for surgical skill assessment. This study determines the feasibility of using metrics computed with object detection by comparing them to widely accepted metrics computed using traditional tracking methods in central venous catheterization. Methods: Both video and tracking data were recorded from participants performing central venous catheterization on a venous access phantom. A Faster Region-Based Convolutional Neural Network was trained to recognize the ultrasound probe and syringe on the video data. Tracking-based metrics were computed using the Perk Tutor extension of 3D Slicer. The path length and usage time for each tool were then computed using both the video and tracking data. The metrics from object detection and tracking were compared using Spearman rank correlation. Results: The path lengths had a rank correlation coefficient of 0.22 for the syringe (p\<0.03) and 0.35 (p\<0.001) for the ultrasound probe. For the usage times, the correlation coefficient was 0.37 (p\<0.001) for the syringe and 0.34 (p\<0.001) for the ultrasound probe. Conclusions: The video-based metrics correlated significantly with the tracked metrics, suggesting that object detection could be a feasible skill assessment method for central venous catheterization.

}, keywords = {central venous catheterization, computer-assisted skill assessment, Object detection}, author = {Olivia O{\textquoteright}Driscoll and R. Hisey and M. Holden and Daenis Camire and Jason Erb and Daniel Howes and Tamas Ungi and Gabor Fichtinger} } @article {ODriscoll2022b, title = {Feasibility of using object detection for performance assessment in central venous catherization}, year = {2022}, month = {02/2022}, author = {Olivia O{\textquoteright}Driscoll and R. Hisey and M. Holden and Daenis Camire and Jason Erb and Daniel Howes and Tamas Ungi and Gabor Fichtinger} } @article {Klosa2022b, title = {Identifying tissues for task recognition in training of open inguinal hernia repairs}, year = {2022}, month = {02/2022}, author = {Elizabeth Klosa and R. Hisey and Tahmina Nazari and Theo Wiggers and Boris Zevin and Tamas Ungi and Gabor Fichtinger} } @conference {Klosa2022a, title = {Tissue segmentation for workflow recognition in open inguinal hernia repair training}, booktitle = {SPIE Medical Imaging}, year = {2022}, month = {02/2022}, publisher = {SPIE Medical Imaging}, organization = {SPIE Medical Imaging}, address = {San Diego}, abstract = {PURPOSE: As medical education adopts a competency-based training method, experts are spending substantial amounts of time instructing and assessing trainees{\textquoteright} competence. In this study, we look to develop a computer-assisted training platform that can provide instruction and assessment of open inguinal hernia repairs without needing an expert observer. We recognize workflow tasks based on the tool-tissue interactions, suggesting that we first need a method to identify tissues. This study aims to train a neural network in identifying tissues in a low-cost phantom as we work towards identifying the tool-tissue interactions needed for task recognition. METHODS: Eight simulated tissues were segmented throughout five videos from experienced surgeons who performed open inguinal hernia repairs on phantoms. A U-Net was trained using leave-one-user-out cross validation. The average F-score, false positive rate and false negative rate were calculated for each tissue to evaluate the U-Net{\textquoteright}s performance. RESULTS: Higher F-scores and lower false negative and positive rates were recorded for the skin, hernia sac, spermatic cord, and nerves, while slightly lower metrics were recorded for the subcutaneous tissue, Scarpa{\textquoteright}s fascia, external oblique aponeurosis and superficial epigastric vessels. CONCLUSION: The U-Net performed better in recognizing tissues that were relatively larger in size and more prevalent, while struggling to recognize smaller tissues only briefly visible. Since workflow recognition does not require perfect segmentation, we believe our U-Net is sufficient in recognizing the tissues of an inguinal hernia repair phantom. Future studies will explore combining our segmentation U-Net with tool detection as we work towards workflow recognition.}, keywords = {open inguinal hernia repair, segmentation, surgical training, U-Net, workflow recognition}, author = {Elizabeth Klosa and R. Hisey and Tahmina Nazari and Theo Wiggers and Boris Zevin and Tamas Ungi and Gabor Fichtinger} } @article {Hisey2021a, title = {Central Line Tutor: using computer vision workflow recognition in a central venous catheterization training system}, year = {2021}, month = {02/2021}, author = {R. Hisey and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger and Tamas Ungi} } @conference {ODriscoll2021a, title = {Object detection to compute performance metrics for skill assessment in central venous catheterization}, booktitle = {SPIE Medical Imaging}, year = {2021}, url = {https://www.spiedigitallibrary.org/conference-proceedings-of-spie/11598/1159816/Object-detection-to-compute-performance-metrics-for-skill-assessment-in/10.1117/12.2581889.short?SSO=1}, author = {Olivia O{\textquoteright}Driscoll and R. Hisey and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger and Tamas Ungi} } @article {ODriscoll2021b, title = {Surgical tool tracking with object detection for performance assessment in central venous catheterization}, year = {2021}, author = {Olivia O{\textquoteright}Driscoll and R. Hisey and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger and Tamas Ungi} } @article {Hisey2021b, title = {System for central venous catheterization training using computer vision-based workflow feedback}, journal = {IEEE Transactions on Biomedical Engineering}, year = {2021}, month = {10/2021}, author = {R. Hisey and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger and Tamas Ungi} } @conference {CBarr2021a, title = {Ultrasound Probe Pose Classification for Task Recognition in Central Venous Catheterization}, booktitle = {Imaging Network of Ontario Symposium}, year = {2021}, month = {02/2021}, author = {Colton Barr and R. Hisey and Tamas Ungi and Gabor Fichtinger} } @conference {CBarr2021b, title = {Ultrasound Probe Pose Classification for Task Recognition in Central Venous Catheterization}, booktitle = {43rd Conference of the IEEE Engineering Medicine and Biology Society}, year = {2021}, month = {10/2021}, abstract = {

Central Line Tutor is a system that facilitates real-time feedback during training for central venous catheterization. One limitation of Central Line Tutor is its reliance on expensive, cumbersome electromagnetic tracking to facilitate various training aids, including ultrasound task identification and segmentation of neck vasculature. The purpose of this study is to validate deep learning methods for vessel segmentation and ultrasound pose classification in order to mitigate the system{\textquoteright}s reliance on electromagnetic tracking. A large dataset of segmented and classified ultrasound images was generated from participant data captured using Central Line Tutor. A U-Net architecture was used to perform vessel segmentation, while a shallow Convolutional Neural Network (CNN) architecture was designed to classify the pose of the ultrasound probe. A second classifier architecture was also tested that used the U-Net output as the CNN input. The mean testing set Intersect over Union score for U-Net cross-validation was 0.746 {\textpm} 0.052. The mean test set classification accuracy for the CNN was 92.0\% {\textpm} 3.0, while the U-Net + CNN achieved 92.7\% {\textpm} 2.1\%. This study highlights the potential for deep learning on ultrasound images to replace the current electromagnetic tracking-based methods for vessel segmentation and ultrasound pose classification, and represents an important step towards removing the electromagnetic tracker altogether. Removing the need for an external tracking system would significantly reduce the cost of Central Line Tutor and make it far more accessible to the medical trainees that would benefit from it most.

}, author = {Colton Barr and R. Hisey and Tamas Ungi and Gabor Fichtinger} } @article {Hisey2020b, title = {Recognizing workflow tasks in central venous catheterization using convolutional neural networks and reinforcement learning}, year = {2020}, month = {06/2020}, pages = {94-95}, author = {R. Hisey and Brian Chen and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger and Tamas Ungi} } @article {Hisey2020a, title = {Reinforcement learning approach for video-based task recognition in central venous catheterization}, year = {2020}, month = {06/2020}, author = {R. Hisey and Brian Chen and Tamas Ungi and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger} } @article {Hisey2019, title = {Comparison of convolutional neural networks for central venous catheterization tool detection}, year = {2019}, month = {03/2019}, address = {Toronto, Ontario}, author = {R. Hisey and Tamas Ungi and Daenis Camire and Jason Erb and Daniel Howes and Gabor Fichtinger} } @article {Isen2019a, title = {Retraining MobileNet with highly variable data for tool detection in central venous catheterization}, year = {2019}, publisher = {Imaging Network Ontario (ImNO)}, address = {London, Ontario}, author = {Jonah Isen and R. Hisey and Tamas Ungi and Gabor Fichtinger} } @article {Isen2019b, title = {Utilizing a convolutional neural network for tool detection in central venous catheterization}, year = {2019}, publisher = {Int J CARS}, address = {Rennes, France}, author = {Jonah Isen and R. Hisey and Tamas Ungi and Gabor Fichtinger} } @article {Hisey2018b, title = {Assessment of the use of webcam based workflow detection for providing real-time feedback in central venous catheterization training}, year = {2018}, author = {R. Hisey and Tamas Ungi and M. Holden and Zachary M C Baum and Keri, Zsuzsanna and Caitlin McCallum and Daniel Howes and Gabor Fichtinger} } @conference {Hisey2018, title = {Real-time workflow detection using webcam video for providing real-time feedback in central venous catheterization training}, booktitle = {SPIE Medical Imaging 2018: Image-Guided Procedures, Robotic Interventions, and Modeling}, year = {2018}, author = {R. Hisey and Tamas Ungi and M. Holden and Zachary M C Baum and Keri, Zsuzsanna and Caitlin McCallum and Daniel Howes and Gabor Fichtinger} }