@conference {Klosa2022a, title = {Tissue segmentation for workflow recognition in open inguinal hernia repair training}, booktitle = {SPIE Medical Imaging}, year = {2022}, month = {02/2022}, publisher = {SPIE Medical Imaging}, organization = {SPIE Medical Imaging}, address = {San Diego}, abstract = {PURPOSE: As medical education adopts a competency-based training method, experts are spending substantial amounts of time instructing and assessing trainees{\textquoteright} competence. In this study, we look to develop a computer-assisted training platform that can provide instruction and assessment of open inguinal hernia repairs without needing an expert observer. We recognize workflow tasks based on the tool-tissue interactions, suggesting that we first need a method to identify tissues. This study aims to train a neural network in identifying tissues in a low-cost phantom as we work towards identifying the tool-tissue interactions needed for task recognition. METHODS: Eight simulated tissues were segmented throughout five videos from experienced surgeons who performed open inguinal hernia repairs on phantoms. A U-Net was trained using leave-one-user-out cross validation. The average F-score, false positive rate and false negative rate were calculated for each tissue to evaluate the U-Net{\textquoteright}s performance. RESULTS: Higher F-scores and lower false negative and positive rates were recorded for the skin, hernia sac, spermatic cord, and nerves, while slightly lower metrics were recorded for the subcutaneous tissue, Scarpa{\textquoteright}s fascia, external oblique aponeurosis and superficial epigastric vessels. CONCLUSION: The U-Net performed better in recognizing tissues that were relatively larger in size and more prevalent, while struggling to recognize smaller tissues only briefly visible. Since workflow recognition does not require perfect segmentation, we believe our U-Net is sufficient in recognizing the tissues of an inguinal hernia repair phantom. Future studies will explore combining our segmentation U-Net with tool detection as we work towards workflow recognition.}, keywords = {open inguinal hernia repair, segmentation, surgical training, U-Net, workflow recognition}, author = {Elizabeth Klosa and R. Hisey and Tahmina Nazari and Theo Wiggers and Boris Zevin and Tamas Ungi and Gabor Fichtinger} } @article {Laframboise2020a, title = {Open source platform for automated collection of training data to support video-based feedback in surgical simulators}, year = {2020}, publisher = {SPIE}, address = {Houston, United States}, abstract = {

Purpose: Surgical training could be improved by automatic detection of workflow steps. A platform to collect and organize tracking and video data would enable rapid development of deep learning solutions for surgical training. The purpose of this research is to demonstrate 3D Slicer / PLUS Toolkit as a platform for video annotation by identifying and annotating tools interacting with tissues in simulated hernia repair. Methods: Tracking data from an optical tracker and video data from a camera are collected by PLUS and 3D Slicer. To demonstrate the platform in use, we identify tissues during a surgical procedure using a neural network. The tracking data is used to identify what tool is in use. The solution is deployed with a custom Slicer module. Results: This platform allowed the collection and storage of enough tracked video data for training a convolutional neural network (CNN) to detect interactions with tissues and tools. The CNN was trained on this data and applied to new data with a testing accuracy of 98\%. The model{\textquoteright}s predictions can be weighted over several frames with a custom Slicer module to improve accuracy. Conclusion: We found the 3D Slicer and PLUS Toolkit platform to be a viable platform for training and deploying a solution that combines automatic video processing and optical tool tracking. We designed a proof of concept model to identify tissues with a trained CNN in real time along with tracking of surgical tools.

}, keywords = {3D Slicer, AI, annotation, collection, data, deep learning, PLUS, surgical training, video}, author = {Jacob Laframboise and Tamas Ungi and Kyle R. Sunderland and Boris Zevin and Gabor Fichtinger} }