publications

@article{nokey,

title = {Modeling of Surgical Procedures Using Statecharts for Semi-Autonomous Robotic Surgery},

author = {Fabio Falezza, Nicola Piccinelli, Giacomo De Rossi, Andrea Roberti, Gernot Kronreif, Francesco Setti,

Paolo Fiorini and Riccardo Muradore},

url = {https://ieeexplore.ieee.org/abstract/document/9530457



https://iris.univr.it/retrieve/handle/11562/1053228/220841/_TMBR__Modelling_of_Surgical_Procedures_using_Statecharts_for_Semi_Autonomous_Robotic_Surgery.pdf



},

doi = {10.1109/TMRB.2021.3110676},

issn = {2576-3202},

year  = {2021},

date = {2021-09-06},

journal = {IEEE Transactions on Medical Robotics and Bionics (special issue for Hamlyn Symposium on Medical Robotics 2020)},

pages = {888 - 899},

abstract = {In this paper we propose a new methodology to model surgical procedures that is specifically tailored to semi-autonomous robotic surgery. We propose to use a restricted version of statecharts to merge the bottom-up approach, based on data-driven techniques (e.g., machine learning), with the top-down approach based on knowledge representation techniques. We consider medical knowledge about the procedure and sensing of the environment in two concurrent regions of the statecharts to facilitate re-usability and adaptability of the modules. Our approach allows producing a well defined procedural model exploiting the hierarchy capability of the statecharts, while machine learning modules act as soft sensors to trigger state transitions. Integrating data driven and prior knowledge techniques provides a robust, modular, flexible and re-configurable methodology to define a surgical procedure which is comprehensible by both humans and machines. We validate our approach on the three surgical phases of a Robot-Assisted Radical Prostatectomy (RARP) that directly involve the assistant surgeon: bladder mobilization, bladder neck transection, and vesicourethral anastomosis, all performed on synthetic manikins.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Daniele2021b,

title = {Unsupervised identification of surgical robotic actions from small non-homogeneous datasets},

author = {Meli Daniele, Fiorini Paolo},

editor = {IEEE},

url = {https://ieeexplore.ieee.org/document/9514470/footnotes#footnotes},

doi = {https://doi.org/10.1109/LRA.2021.3104880},

issn = {2377-3766},

year  = {2021},

date = {2021-08-16},

journal = {IEEE Robotics and Automation Letters},

volume = {6},

number = {4},

pages = {8205 - 8212},

abstract = {Robot-assisted surgery is an established clinical practice. The automatic identification of surgical actions is needed for a range of applications, including performance assessment of trainees and surgical process modeling for autonomous execution and monitoring. However, supervised action identification is not feasible, due to the burden of manually annotating recordings of potentially complex and long surgical executions. Moreover, often few example executions of a surgical procedure can be recorded. This letter proposes a novel fast algorithm for unsupervised identification of surgical actions in a standard surgical training task, the ring transfer, executed with da Vinci Research Kit. Exploiting kinematic and semantic visual features automatically extracted from a very limited dataset of executions, we are able to significantly outperform state-of-the-art results on a dataset of non-expert executions (58% vs. 24% F1-score), and improve performance in the presence of noise, short actions and non-homogeneous workflows, i.e. non repetitive action sequences.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Daniele2021c,

title = {Unsupervised Identification of Surgical Robotic Actions From Small Non-Homogeneous Datasets},

author = {Meli Daniele, Fiorini Paolo},

editor = {IEEE},

url = {https://ieeexplore.ieee.org/document/9514470



https://arxiv.org/abs/2105.08488},

doi = {10.1109/LRA.2021.3104880},

issn = {2377-3766},

year  = {2021},

date = {2021-08-16},

urldate = {2021-08-16},

journal = {Robotics and Automation Letters},

volume = {6},

number = {October 2021},

issue = {4},

pages = {8205-8212},

abstract = {Robot-assisted surgery is an established clinical practice. The automatic identification of surgical actions is needed for a range of applications, including performance assessment of trainees and surgical process modeling for autonomous execution and monitoring. However, supervised action identification is not feasible, due to the burden of manually annotating recordings of potentially complex and long surgical executions. Moreover, often few example executions of a surgical procedure can be recorded. This paper proposes a novel fast algorithm for unsupervised identification of surgical actions in a standard surgical training task, the ring transfer, executed with da Vinci Research Kit. Exploiting kinematic and semantic visual features automatically extracted from a very limited dataset of executions, we are able to significantly outperform state-of-the-art results on a dataset of non-expert executions (58% vs. 24% F1-score), and improve performance in the presence of noise, short actions and non-homogeneous workflows, i.e. non repetitive action sequences.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Ginesi2021b,

title = {Overcoming Some Drawbacks of Dynamic Movement Primitives},

author = {Michele Ginesi, Nicola Sansonetto, Paolo Fiorini},

url = {https://www.sciencedirect.com/science/article/abs/pii/S0921889021001299



https://arxiv.org/abs/1908.10608},

doi = {https://doi.org/10.1016/j.robot.2021.103844},

year  = {2021},

date = {2021-07-08},

urldate = {2021-07-08},

journal = {Robotics and Autonomous Systems},

volume = {144},

abstract = {Dynamic Movement Primitives (DMPs) is a framework for learning a point-to-point trajectory from a demonstration. Despite being widely used, DMPs still present some shortcomings that may limit their usage in real robotic applications. Firstly, at the state of the art, mainly Gaussian basis functions have been used to perform function approximation. Secondly, the adaptation of the trajectory generated by the DMP heavily depends on the choice of hyperparameters and the new desired goal position. Lastly, DMPs are a framework for ‘one-shot learning’, meaning that they are constrained to learn from a unique demonstration. In this work, we present and motivate a new set of basis functions to be used in the learning process, showing their ability to accurately approximate functions while having both analytical and numerical advantages w.r.t. Gaussian basis functions. Then, we show how to use the invariance of DMPs w.r.t. affine transformations to make the generalization of the trajectory robust against both the choice of hyperparameters and new goal position, performing both synthetic tests and experiments with real robots to show this increased robustness. Finally, we propose an algorithm to extract a common behavior from multiple observations, validating it both on a synthetic dataset and on a dataset obtained by performing a task on a real robot.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Daniele2021,

title = {Inductive learning of answer set programs for autonomous surgical task planning - application to a training task for surgeons},

author = {Meli Daniele, Sridharan Mohan, Fiorini Paolo},

editor = {Springer},

url = {https://link.springer.com/article/10.1007/s10994-021-06013-7?wt_mc=Internal.Event.1.SEM.ArticleAuthorOnlineFirst&utm_source=ArticleAuthorOnlineFirst&utm_medium=email&utm_content=AA_en_06082018&ArticleAuthorOnlineFirst_20210616#article-info},

doi = {https://doi.org/10.1007/s10994-021-06013-7},

issn = {0885-6125},

year  = {2021},

date = {2021-06-15},

journal = {Machine Learning},

abstract = {The quality of robot-assisted surgery can be improved and the use of hospital resources can be optimized by enhancing autonomy and reliability in the robot’s operation. Logic programming is a good choice for task planning in robot-assisted surgery because it supports reliable reasoning with domain knowledge and increases transparency in the decision making. However, prior knowledge of the task and the domain is typically incomplete, and it often needs to be refined from executions of the surgical task(s) under consideration to avoid sub-optimal performance. In this paper, we investigate the applicability of inductive logic programming for learning previously unknown axioms governing domain dynamics. We do so under answer set semantics for a benchmark surgical training task, the ring transfer. We extend our previous work on learning the immediate preconditions of actions and constraints, to also learn axioms encoding arbitrary temporal delays between atoms that are effects of actions under the event calculus formalism. We propose a systematic approach for learning the specifications of a generic robotic task under the answer set semantics, allowing easy knowledge refinement with iterative learning. In the context of 1000 simulated scenarios, we demonstrate the significant improvement in performance obtained with the learned axioms compared with the hand-written ones; specifically, the learned axioms address some critical issues related to the plan computation time, which is promising for reliable real-time performance during surgery.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Ginesi2021,

title = {Dynamic Movement Primitives: Volumetric Obstacle Avoidance Using Dynamic Potential Functions},

author = {Michele Ginesi, Daniele Meli, Andrea Roberti, Nicola Sansonetto, Paolo Fiorini},

editor = {Topical collection on ICAR 2019 Special Issue},

url = {https://link.springer.com/article/10.1007/s10846-021-01344-y

https://iris.univr.it/handle/11562/1042079},

doi = {http://dx.doi.org/10.1007/s10846-021-01344-y},

isbn = {1573-0409},

year  = {2021},

date = {2021-04-28},

journal = {Journal of Intelligent & Robotic Systems},

volume = {issue 101(4)},

number = {Article n° 79},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Bombieri2021,

title = {Automatic detection of procedural knowledge in robotic-assisted surgical texts},

author = {Marco Bombieri, Marco Rospocher, Diego Dall’Alba, Paolo Fiorini},

editor = {Springer},

url = {https://link.springer.com/article/10.1007/s11548-021-02370-9

https://iris.univr.it/handle/11562/1042459?mode=full.40#.YJlNv7UzaUk},

doi = {https://doi.org/10.1007/s11548-021-02370-9},

year  = {2021},

date = {2021-04-22},

journal = {International Journal of Computer Assisted Radiology and Surgery},

abstract = {Purpose



The automatic extraction of knowledge about intervention execution from surgical manuals would be of the utmost importance to develop expert surgical systems and assistants. In this work we assess the feasibility of automatically identifying the sentences of a surgical intervention text containing procedural information, a subtask of the broader goal of extracting intervention workflows from surgical manuals.

Methods



We frame the problem as a binary classification task. We first introduce a new public dataset of 1958 sentences from robotic surgery texts, manually annotated as procedural or non-procedural. We then apply different classification methods, from classical machine learning algorithms, to more recent neural-network approaches and classification methods exploiting transformers (e.g., BERT, ClinicalBERT). We also analyze the benefits of applying balancing techniques to the dataset.

Results



The architectures based on neural-networks fed with FastText’s embeddings and the one based on ClinicalBERT outperform all the tested methods, empirically confirming the feasibility of the task. Adopting balancing techniques does not lead to substantial improvements in classification.

Conclusion



This is the first work experimenting with machine / deep learning algorithms for automatically identifying procedural sentences in surgical texts. It also introduces the first public dataset that can be used for benchmarking different classification methods for the task. },

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{tagliabue2021data,

title = {Data-driven Intra-operative Estimation of Anatomical Attachments for Autonomous Tissue Dissection},

author = {Eleonora Tagliabue and Diego DallÁlba and Micha Pfeiffer and Marco Piccinelli and Riccardo Marin and Umberto Castellani and Stefanie Speidel and Paolo Fiorini},

editor = {IEEE},

doi = {10.1109/LRA.2021.3060655},

year  = {2021},

date = {2021-01-01},

journal = {IEEE Robotics and Automation Letters},

volume = {6},

number = {Issue: 2, April 2021},

pages = {1856 - 1863},

publisher = {IEEE},

abstract = {The execution of surgical tasks by an Autonomous Robotic System (ARS) requires an up-to-date model of the current surgical environment, which has to be deduced from measurements collected during task execution. In this work, we propose to automate tissue dissection tasks by introducing a convolutional neural network, called BA-Net, to predict the location of attachment points between adjacent tissues. BA-Net identifies the attachment areas from a single partial view of the deformed surface, without any a-priori knowledge about their location. The proposed method guarantees a very fast prediction time, which makes it ideal for intra-operative applications. Experimental validation is carried out on both simulated and real world phantom data of soft tissue manipulation performed with the da Vinci Research Kit (dVRK). The obtained results demonstrate that BA-Net provides robust predictions at varying geometric configurations, material properties, distributions of attachment points and grasping point locations. The estimation of attachment points provided by BA-Net improves the simulation of the anatomical environment where the system is acting, leading to a median simulation error below 5 mm in all the tested conditions. BA-Net can thus further support an ARS by providing a more robust test bench for the robotic actions intra-operatively, in particular when replanning is needed. The method and collected dataset are available at https://gitlab.com/altairLab/banet .

},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Roberti2020,

title = {Improving rigid 3D calibration for robotic surgery},

author = {Andrea Roberti, Nicola Piccinelli, Daniele Meli, Riccardo Muradore, Paolo Fiorini},

editor = {IEEE Transactions on Medical Robotics and Bionics (special issue for Hamlyn Symposium on Medical Robotics 2020)},

url = {https://ieeexplore.ieee.org/document/9239343},

doi = {https://doi.org/10.1109/TMRB.2020.3033670},

isbn = {2576-3202 },

year  = {2020},

date = {2020-10-26},

journal = {IEEE Transactions on Medical Robotics and Bionics (special issue for Hamlyn Symposium on Medical Robotics 2020)},

pages = {569 - 573},

abstract = {Autonomy is the next frontier of research in robotic surgery and its aim is to improve the quality of surgical procedures in the next future. One fundamental requirement for autonomy is advanced perception capability through vision sensors. In this article, we propose a novel calibration technique for a surgical scenario with a da Vinci ® Research Kit (dVRK) robot. Camera and robotic arms calibration are necessary to precise position and emulate expert surgeon. The novel calibration technique is tailored for RGB-D cameras. Different tests performed on relevant use cases prove that we significantly improve precision and accuracy with respect to state of the art solutions for similar devices on a surgical-size setups. Moreover, our calibration method can be easily extended to standard surgical endoscope used in real surgical scenario.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Meli2020,

title = {Towards inductive learning of surgical task knowledge: a preliminary case study of the peg transfer task},

author = {Daniele Meli, Paolo Fiorini, Mohan Sridharan},

editor = {Procedia of Computer Science - special issue Knowledge-Based and Intelligent Information & Engineering Systems: Proceedings of the 24th International Conference KES2020},

url = {http://hdl.handle.net/11562/1027998},

doi = {https://doi.org/10.1016/j.procs.2020.08.046},

isbn = {1877-0509},

year  = {2020},

date = {2020-06-17},

journal = {Procedia Computer Science},

pages = {440-449},

abstract = {Autonomy in robotic surgery will significantly improve the quality of interventions in terms of safety and recovery time for the patient, and reduce fatigue of surgeons and hospital costs. A key requirement for such autonomy is the ability of the surgical system to encode and reason with commonsense task knowledge, and to adapt to variations introduced by the surgical scenarios and the individual patients. However, it is difficult to encode all the variability in surgical scenarios and in the anatomy of individual patients a priori, and new knowledge often needs to be acquired and merged with the existing knowledge. At the same time, it is not possible to provide a large number of labeled training examples in the robotic surgery. This paper presents a framework based on inductive logic programming and answer set semantics for incrementally learning domain knowledge from a limited number of executions of basic surgical tasks. As an illustrative example, we focus on the peg transfer task, and learn state constraints and the preconditions of actions starting from different levels of prior knowledge. We do so using a small dataset comprising human and robotic executions with the da Vinci surgical robot in a challenging simulated scenario.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Tagliabue2020,

title = {Biomechanical modelling of probe to tissue interaction during ultrasound scanning},

author = {Eleonora Tagliabue, Diego Dall’Alba, Enrico Magnabosco, Igor Peterlik, Paolo Fiorini},

editor = {Springer},

url = {https://link.springer.com/article/10.1007/s11548-020-02183-2?wt_mc=Internal.Event.1.SEM.ArticleAuthorOnlineFirst&utm_source=ArticleAuthorOnlineFirst&utm_medium=email&utm_content=AA_en_06082018&ArticleAuthorOnlineFirst_20200523#citeas},

doi = {https://doi.org/10.1007/s11548-020-02183-2},

year  = {2020},

date = {2020-05-22},

journal = {International Journal of Computer Assisted Radiology and Surgery},

abstract = {Purpose



Biomechanical simulation of anatomical deformations caused by ultrasound probe pressure is of outstanding importance for several applications, from the testing of robotic acquisition systems to multi-modal image fusion and development of ultrasound training platforms. Different approaches can be exploited for modelling the probe–tissue interaction, each achieving different trade-offs among accuracy, computation time and stability.

Methods



We assess the performances of different strategies based on the finite element method for modelling the interaction between the rigid probe and soft tissues. Probe–tissue contact is modelled using (i) penalty forces, (ii) constraint forces, and (iii) by prescribing the displacement of the mesh surface nodes. These methods are tested in the challenging context of ultrasound scanning of the breast, an organ undergoing large nonlinear deformations during the procedure.

Results



The obtained results are evaluated against those of a non-physically based method. While all methods achieve similar accuracy, performance in terms of stability and speed shows high variability, especially for those methods modelling the contacts explicitly. Overall, prescribing surface displacements is the approach with best performances, but it requires prior knowledge of the contact area and probe trajectory.

Conclusions



In this work, we present different strategies for modelling probe–tissue interaction, each able to achieve different compromises among accuracy, speed and stability. The choice of the preferred approach highly depends on the requirements of the specific clinical application. Since the presented methodologies can be applied to describe general tool–tissue interactions, this work can be seen as a reference for researchers seeking the most appropriate strategy to model anatomical deformation induced by the interaction with medical tools.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{menegozzo2019automatic,

title = {Automatic process modeling with time delay neural network based on low-level data.},

author = {Giovanni Menegozzo and Diego Dall’Alba and Andrea Roberti and Paolo Fiorini},

url = {https://www.sciencedirect.com/science/article/pii/S2351978920300172/pdf?md5=92de1e9af7b7717c4a79e0a7ab8872f4&pid=1-s2.0-S2351978920300172-main.pdf},

year  = {2019},

date = {2019-06-24},

journal = {Procedia Manufacturing},

volume = {38},

pages = {125--132},

publisher = {Elsevier},

abstract = {Automatic process modelling (APM) is an enabling technology for the development of intelligent manufacturingsystems (IMSs). The analysis of obtained models enables the prompt detection of error-prone steps and the design of proper mitigation strategies, in all aspects of the manufacturing process, from parameter optimization to development of customized personnel training. In this work we propose a Time Delay Neural Network (TDNN) applied to low level data for the automatic recognition of different process phases in industrial collaborative tasks.

We selected TDNN because they are suited for modelling time dependent processes over long sequences while maintaining computational efficiency. To experimentally evaluate the recognition performance and the generalization capability of the proposed method, we acquired two novel datasets reproducing a typical IMS setting. Datasets (including manually annotated ground-truth labels) are publicly available to enable other methods to be tested on them and they replicate typical Industry 4.0 setting. The first dataset replicates a collaborative robotic environment where a human operator interacts with a robotic manipulator in the execution of a pick and place task. 

The second set represents a human tele-operated robotic assisted manipulation for assembly applications. The obtained results are superior to other methods available in literature and demonstrate an improved computational performance.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{tagliabue2019positionb,

title = {Position-based modeling of lesion displacement in Ultrasound-guided breast biopsy},

author = {Eleonora Tagliabue and Diego Dall’Alba and Enrico Magnabosco and Chiara Tenga and Igor Peterl{'i}k and Paolo Fiorini},

url = {https://link.springer.com/article/10.1007/s11548-019-01997-z},

doi = {https://doi.org/10.1007/s11548-019-01997-z},

year  = {2019},

date = {2019-01-01},

journal = {International journal of computer assisted radiology and surgery},

volume = {14},

number = {8},

pages = {1329--1339},

publisher = {Springer},

abstract = {Purpose



Although ultrasound (US) images represent the most popular modality for guiding breast biopsy, malignant regions are often missed by sonography, thus preventing accurate lesion localization which is essential for a successful procedure. Biomechanical models can support the localization of suspicious areas identified on a preoperative image during US scanning since they are able to account for anatomical deformations resulting from US probe pressure. We propose a deformation model which relies on position-based dynamics (PBD) approach to predict the displacement of internal targets induced by probe interaction during US acquisition.





Methods



The PBD implementation available in NVIDIA FleX is exploited to create an anatomical model capable of deforming online. Simulation parameters are initialized on a calibration phantom under different levels of probe-induced deformations; then, they are fine-tuned by minimizing the localization error of a US–visible landmark of a realistic breast phantom. The updated model is used to estimate the displacement of other internal lesions due to probe-tissue interaction.





Results



The localization error obtained when applying the PBD model remains below 11 mm for all the tumors even for input displacements in the order of 30 mm. This proposed method obtains results aligned with FE models with faster computational performance, suitable for real-time applications. In addition, it outperforms rigid model used to track lesion position in US-guided breast biopsies, at least halving the localization error for all the displacement ranges considered.

Conclusion



Position-based dynamics approach has proved to be successful in modeling breast tissue deformations during US acquisition. Its stability, accuracy and real-time performance make such model suitable for tracking lesions displacement during US-guided breast biopsy.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{cheng2019design,

title = {Design and integration of electrical bio-impedance sensing in surgical robotic tools for tissue identification},

author = {Z. Cheng, D. Dall'Alba, S. Foti, A. Mariani, T. Chupin, D. Caldwell, P. Fiorini, E. De Momi, G. Ferrigno and L. Mattos},

doi = {http://hdl.handle.net/11562/1018566},

year  = {2019},

date = {2019-01-01},

journal = {8th joint workshop on New Technologies for Computer/Robot Assisted Surgery},

volume = {6},

pages = {55},

publisher = {Frontiers},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{cheng2019designc,

title = {Design and integration of electrical bio-impedance sensing in surgical robotic tools for tissue identification and display},

author = {Zhuoqi Cheng and Diego DallÁlba and Simone Foti and Andrea Mariani and Thibaud Jean Eudes Chupin and Darwin Gordon Caldwell and Giancarlo Ferrigno and Elena De Momi and Leonardo S Mattos and Paolo Fiorini},

doi = {10.3389/frobt.2019.00055},

isbn = {22969144},

year  = {2019},

date = {2019-01-01},

journal = {Frontiers in Robotics and AI},

volume = {6},

pages = {55},

publisher = {Frontiers},

abstract = {The integration of intra-operative sensors into surgical robots is a hot research topic since this can significantly facilitate complex surgical procedures by enhancing surgical awareness with real-time tissue information. However, currently available intra-operative sensing technologies are mainly based on image processing and force feedback, which normally require heavy computation or complicated hardware modifications of existing surgical tools. This paper presents the design and integration of electrical bio-impedance sensing into a commercial surgical robot tool, leading to the creation of a novel smart instrument that allows the identification of tissues by simply touching them. In addition, an advanced user interface is designed to provide guidance during the use of the system and to allow augmented-reality visualization of the tissue identification results. The proposed system imposes minor hardware modifications to an existing surgical tool, but adds the capability to provide a wealth of data about the tissue being manipulated. This has great potential to allow the surgeon (or an autonomous robotic system) to better understand the surgical environment. To evaluate the system, a series of ex-vivo experiments were conducted. The experimental results demonstrate that the proposed sensing system can successfully identify different tissue types with 100% classification accuracy. In addition, the user interface was shown to effectively and intuitively guide the user to measure the electrical impedance of the target tissue, presenting the identification results as augmented-reality markers for simple and immediate recognition.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}