<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JSG</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Serious Games</journal-id>
      <journal-title>JMIR Serious Games</journal-title>
      <issn pub-type="epub">2291-9279</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v9i2e26820</article-id>
      <article-id pub-id-type="pmid">33769295</article-id>
      <article-id pub-id-type="doi">10.2196/26820</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Virtual Reality Human–Human Interface to Deliver Psychotherapy to People Experiencing Auditory Verbal Hallucinations: Development and Usability Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Zary</surname>
            <given-names>Nabil</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Brooke</surname>
            <given-names>John</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Lundin</surname>
            <given-names>Robert</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Brander</surname>
            <given-names>Mischa</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0572-2964</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Egger</surname>
            <given-names>Stephan T</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <address>
            <institution>Psychotherapy and Psychosomatics</institution>
            <institution>Department of Psychiatry</institution>
            <institution>Psychiatric University Hospital of Zurich, University of Zürich</institution>
            <addr-line>Lenggstrasse 31</addr-line>
            <addr-line>Zurich, 8032</addr-line>
            <country>Switzerland</country>
            <phone>41 523049340</phone>
            <email>stephan.egger@pukzh.ch</email>
          </address>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0314-4929</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Hürlimann</surname>
            <given-names>Noa</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4780-6813</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Seifritz</surname>
            <given-names>Erich</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7311-4426</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Sumner</surname>
            <given-names>Robert W</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1909-8082</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Vetter</surname>
            <given-names>Stefan</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3063-0965</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Magnenat</surname>
            <given-names>Stéphane</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4367-4588</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Game Technology Center</institution>
        <institution>Department of Computer Science</institution>
        <institution>Eidgenössische Technische Hochschule Zürich</institution>
        <addr-line>Zurich</addr-line>
        <country>Switzerland</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Psychotherapy and Psychosomatics</institution>
        <institution>Department of Psychiatry</institution>
        <institution>Psychiatric University Hospital of Zurich, University of Zürich</institution>
        <addr-line>Zurich</addr-line>
        <country>Switzerland</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Psychiatry</institution>
        <institution>Faculty of Medicine</institution>
        <institution>University of Oviedo</institution>
        <addr-line>Oviedo</addr-line>
        <country>Spain</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Stephan T Egger <email>stephan.egger@pukzh.ch</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <season>Apr-Jun</season>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>1</day>
        <month>6</month>
        <year>2021</year>
      </pub-date>
      <volume>9</volume>
      <issue>2</issue>
      <elocation-id>e26820</elocation-id>
      <history>
        <date date-type="received">
          <day>29</day>
          <month>12</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>8</day>
          <month>2</month>
          <year>2021</year>
        </date>
        <date date-type="rev-recd">
          <day>5</day>
          <month>3</month>
          <year>2021</year>
        </date>
        <date date-type="accepted">
          <day>19</day>
          <month>3</month>
          <year>2021</year>
        </date>
      </history>
      <copyright-statement>©Mischa Brander, Stephan T Egger, Noa Hürlimann, Erich Seifritz, Robert W Sumner, Stefan Vetter, Stéphane Magnenat. Originally published in JMIR Serious Games (https://games.jmir.org), 01.06.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Serious Games, is properly cited. The complete bibliographic information, a link to the original publication on https://games.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://games.jmir.org/2021/2/e26820" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Digital technologies have expanded the options for delivering psychotherapy, permitting for example, the treatment of schizophrenia using Avatar Therapy. Despite its considerable potential, this treatment method has not been widely disseminated. As a result, its operability and functionality remain largely unknown.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>We aimed to study the usability of a therapeutic virtual reality human–human interface, created in a game engine.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>Participants were psychiatric hospital staff who were introduced to the therapeutic platform in a hands-on session. The System Usability Scale (SUS) was employed for evaluation purposes. Statistical evaluation was conducted using descriptive statistics, the chi-square test, analysis of variance, and multilevel factor analysis.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>In total, 109 staff members were introduced to the therapeutic tool and completed the SUS. The mean SUS global score was 81.49 (SD 11.1). Psychotherapists (mean 86.44, SD 8.79) scored significantly higher (<italic>F</italic><sub>2,106</sub>=6.136; <italic>P</italic>=.003) than nursing staff (mean 79.01, SD 13.30) and administrative personnel (mean 77.98, SD 10.72). A multilevel factor analysis demonstrates a different factor structure for each profession.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>In all professional groups in this study, the usability of a digital psychotherapeutic tool developed using a game engine achieved the benchmark for an excellent system, scoring highest among the professional target group (psychotherapists). The usability of the system seems, to some extent, to be dependent on the professional background of the user. It is possible to create and customize novel psychotherapeutic approaches with gaming technologies and platforms.</p>
        </sec>
        <sec sec-type="Trial Registration">
          <title>Trial Registration</title>
          <p>Clinicaltrials.gov NCT04099940; http://clinicaltrials.gov/ct2/show/NCT04099940</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>system usability</kwd>
        <kwd>virtual reality psychotherapy</kwd>
        <kwd>verbal auditory hallucinations</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Psychotherapy is an effective and cost-efficient method for the treatment of psychiatric and psychological disorders [<xref ref-type="bibr" rid="ref1">1</xref>]. Over the last few decades, it has been evolving continuously, demonstrating both feasibility and efficacy in practically all diagnostic categories. Indeed, in several categories, it has become the first line of treatment [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref3">3</xref>]. In patients with schizophrenia, psychotherapy as a treatment option has been largely neglected. Recently, however, it has gained recognition as an effective treatment when used in conjunction with pharmacotherapy [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>]. Furthermore, current guidelines now recommend the early implementation of psychotherapy in the treatment process [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>].</p>
      <p>Psychotherapeutic treatment using digital technologies, virtual reality in particular, has been shown to be at least as efficacious as other treatments [<xref ref-type="bibr" rid="ref8">8</xref>]. In some fields, particularly schizophrenia, digital technologies have considerably extended therapeutic options [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref10">10</xref>], for example, with the novel implementation of Avatar Therapy, whereby psychotherapy is delivered through a computer interface [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. Patients with auditory verbal hallucinations create an avatar of a human entity, to which they attribute the voices. With the help of a therapist, they progressively gain control over the voices, which leads to a reduction of symptoms and distress while increasing quality of life [<xref ref-type="bibr" rid="ref12">12</xref>].</p>
      <p>Despite encouraging early studies and its vast potential, psychotherapeutic treatment using digital technologies has still not been widely disseminated in research or clinical practice [<xref ref-type="bibr" rid="ref10">10</xref>]. We attribute the limited deployment partially to unavailability as off the shelf tools, making implementation difficult [<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref14">14</xref>]. From previous research, it is known that for the optimal delivery of therapy through digital technologies, besides availability, the operability and functionality of the technology are crucial; only once these are well established can the therapist confidently utilize digital technology [<xref ref-type="bibr" rid="ref14">14</xref>,<xref ref-type="bibr" rid="ref15">15</xref>]. Moreover, the proper use of such technology is essential for the optimal delivery of the therapy, thus allowing the therapist to develop their therapeutic skills [<xref ref-type="bibr" rid="ref10">10</xref>].</p>
      <p>In this paper, we present a human–human interface that we developed for use in the treatment of patients experiencing verbal acoustic hallucinations. As the usability of the system is a prerequisite for its clinical application, we systematically sought input from nonpatient users [<xref ref-type="bibr" rid="ref16">16</xref>]. Within mixed skill-grade users, we sought to determine what influence professional background and therapeutic skills have in relation to the use of the therapeutic system.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Virtual Reality Human–Human Interface</title>
        <p>Building upon previous studies [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>], we have created a virtual reality human–human interface using the Unity game engine (Unity Technologies) to deliver Avatar Therapy for people experiencing auditory verbal hallucinations. The basic design employs 2 separate apps running on different devices connected via a network, including bidirectional audio (full-duplex voice over internet protocol connection) communication. The first computer hosts a personal avatar creation tool (Virtual Reality Avatar-Creation Tool, VRAT-CT) to design and customize a humanoid avatar, to which patients attribute their auditory verbal hallucinations. This computer also renders the virtual reality through a head-mounted display for the therapeutic session. The voice of the therapist is modulated through a voice transformer (Roland VT-4) to match the auditory verbal hallucination. The therapeutic session is initialized and controlled from the second computer with a Control Center (VRAT-CC) which allows the therapist to control the Avatar and to speak through the Avatar using lip synchronization.</p>
      </sec>
      <sec>
        <title>Software and Hardware</title>
        <p>The Unity game engine is a freely available platform for game development. Over 60% of current virtual and augmented reality content have been created with Unity [<xref ref-type="bibr" rid="ref19">19</xref>]. It provides a 3D editor, a scripting application programming interface written in C# which allows all components to be brought together, and supports 3D graphics, socket communication, and virtual reality (VR). Both apps are created with Unity (version 2019.3.7) and the associated scripts are written in C# with Microsoft Visual Studio 2017, an integrated development environment. For the 3D character, the Multipurpose Avatar package (version 2.11.5; Unity Technologies) was used. To increase the impression that the avatar is speaking, the SALSA LipSync Suite package (version 2.5.0.92; Crazy Minnow Studio LLC) is used to synchronize lip and mouth movements to the voice input of the therapist.</p>
        <p>The Reverb (Hewlett Packard) headset is used as a head-mounted display. The headset provides a resolution of 2160×2160 per eye at 90 Hz and with a 114° field of view; however, this comes with a series of minimum computational requirements. The producer recommends, at minimum, a Nvidia GeForce GTX 1080 graphic card or an AMD Radeon Pro WX 8200, an Intel Core i7 processor, and 16 GB of RAM. For the operating system, Windows 10 (version 1809 or later) is needed. For this project, a Roland VT-4 voice transformer was used, which provided a set of options for manipulating a voice in real-time. Pitch and format frequency, which can be set with sliders for a deeper or higher voice, were relevant for creating the avatar voice. <xref rid="figure1" ref-type="fig">Figure 1</xref> shows an overview of the software and hardware setup (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Virtual reality (VR) human–human interface.</p>
          </caption>
          <graphic xlink:href="games_v9i2e26820_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>System Usability Scale</title>
        <p>The System Usability Scale (SUS) is a tool for measuring the usability of a wide variety of products and services including hardware, software, mobile devices, websites, and apps [<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref20">20</xref>]. It is a 10-item questionnaire, with a 5-point Likert Scale from 1 (strongly disagree) to 5 (strongly agree) (<xref ref-type="boxed-text" rid="box1">Textbox 1</xref>). Scale items alternate between positive and negative statements; therefore, correction is required for scoring. For odd-numbered items, the value 1 is subtracted from the user's response whereas for even-numbered items, the user's response is subtracted from 5, yielding a score from 0 to 4 for each item. For interpretation, scores are summed and multiplied by a factor of 2.5. The final score ranges from 0 to 100 [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
        <boxed-text id="box1" position="float">
          <title>System Usability Scale.</title>
          <list list-type="order">
            <list-item>
              <p>I think that I would like to use this system frequently.</p>
            </list-item>
            <list-item>
              <p>I found the system unnecessarily complex.</p>
            </list-item>
            <list-item>
              <p>I thought the system was easy to use.</p>
            </list-item>
            <list-item>
              <p>I think that I would need the support of a technical person to be able to use this system.</p>
            </list-item>
            <list-item>
              <p>I found the various functions in this system were well integrated.</p>
            </list-item>
            <list-item>
              <p>I thought there was too much inconsistency in this system.</p>
            </list-item>
            <list-item>
              <p>I would imagine that most people would learn to use this system very quickly.</p>
            </list-item>
            <list-item>
              <p>I found the system very cumbersome to use.</p>
            </list-item>
            <list-item>
              <p>I felt very confident using the system.</p>
            </list-item>
            <list-item>
              <p>I needed to learn a lot of things before I could get going with this system.</p>
            </list-item>
          </list>
        </boxed-text>
      </sec>
      <sec>
        <title>Participants and Assessment</title>
        <p>Employees (irrespective of professional background and occupation) of the Psychiatric University Hospital of Zürich were invited to view and test the virtual reality human–human interface used to deliver Avatar Therapy to people experiencing auditory verbal hallucinations. Basic demographic characteristics (age, gender, and occupation) were gathered. Participants were divided into 3 categories: psychotherapists (either psychiatrists or psychologists), nursing staff, and administrative personnel.</p>
      </sec>
      <sec>
        <title>Procedure</title>
        <p>Participants were individually informed about the nature of the study and introduced to the therapeutic platform in a hands-on session. They were provided with information about the theoretical background of the therapy and the design and implementation process of the virtual reality human–human interface in practice. Each step of the therapeutic process was explained. Afterward, they created an avatar and customized its voice before experiencing it through VR. Thus, they first created and customized their Avatar for therapy in the patient’s role, after which they carried out a session in the therapeutic role. After the session, participants completed the SUS for each component.</p>
      </sec>
      <sec>
        <title>Statistical Analysis</title>
        <p>Descriptive statistics (percentages, means, standard deviations) were used to represent the demographic characteristics of the sample. Differences in the sample were calculated using the chi-square test for proportions. An analysis of variance was performed on continuous variables. The SUS score for the system was calculated. Scores for the avatar creation tool (VRAT-CT) and the control center (VRAT-CC) were evaluated separately. The SUS was evaluated at both item level and global level.</p>
        <p>Additionally, a multilevel factor analysis was conducted. Statistical analysis was performed using the statistical language program (version 4.0.3, The R Project).</p>
      </sec>
      <sec>
        <title>Ethics</title>
        <p>The study was designed to comply with current ethical standards and local regulations. The ethics committee of the Canton of Zürich approved the study protocol (BASEC 2019-01386). The study was registered at Clinicaltrials.gov (NCT04099940).</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Sample Demographics</title>
        <p>In total, 109 staff members were introduced to the therapy. The sample comprised psychotherapists (n=40), nursing staff (n=43), and administrative personnel (n=26); with a mean age of 34.76 (SD 12.69); 74 participants were female (67.9%). There were no statistically significant differences regarding age or gender distribution among the different professions (<xref ref-type="table" rid="table1">Table 1</xref>).</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Sample characteristics and outcome evaluation.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="180"/>
            <col width="0"/>
            <col width="190"/>
            <col width="0"/>
            <col width="150"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="0"/>
            <col width="140"/>
            <col width="0"/>
            <col width="70"/>
            <thead>
              <tr valign="top">
                <td colspan="3">Results</td>
                <td colspan="7">Profession</td>
                <td colspan="3">Statistics</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td colspan="2">Psychotherapists (n=40)</td>
                <td colspan="2">Nursing staff (n=43)</td>
                <td colspan="2">Administrative personnel (n=26)</td>
                <td colspan="3"><italic>F</italic> test (<italic>df1,df2</italic>)</td>
                <td colspan="2"><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="3">Age, mean (SD)</td>
                <td colspan="2">33.25 (9.00)</td>
                <td colspan="2">33.51 (15.85)</td>
                <td colspan="2">30.15 (10.60)</td>
                <td colspan="3">2.09 (2, 106)</td>
                <td>.13</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Gender, n (%)</bold>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="3">4.451 (2, 109)<sup>a</sup></td>
                <td>.11</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Male</td>
                <td colspan="2">19 (47)</td>
                <td colspan="2">12 (28)</td>
                <td colspan="2">7 (27)</td>
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Female</td>
                <td colspan="2">21 (53)</td>
                <td colspan="2">31 (72)</td>
                <td colspan="2">19 (73)</td>
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>SUS<sup>b</sup> score, mean (SD)</bold>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="3">
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Global</td>
                <td colspan="2">86.44 (8.79)<sup>c</sup></td>
                <td colspan="2">79.01 (13.30)</td>
                <td colspan="2">77.98 (10.72)</td>
                <td colspan="3">6.136 (2, 106)</td>
                <td colspan="2">.003</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Virtual reality avatar</td>
                <td colspan="2">87.00 (9.83)<sup>c</sup></td>
                <td colspan="2">79.71 (13.56)</td>
                <td colspan="2">78.08 (12.50)</td>
                <td colspan="3">5.597 (2, 106)</td>
                <td colspan="2">.005</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Control center</td>
                <td colspan="2">85.88 (9.53)<sup>c</sup></td>
                <td colspan="2">78.31 (14.54)</td>
                <td colspan="2">77.88 (11.68)</td>
                <td colspan="3">5.064 (2, 106)</td>
                <td colspan="2">.008</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>Chi-square test statistic (<italic>df1</italic>,<italic>df2</italic>).</p>
            </fn>
            <fn id="table1fn2">
              <p><sup>b</sup>SUS: System Usability Scale.</p>
            </fn>
            <fn id="table1fn3">
              <p><sup>c</sup>Posthoc analysis with Bonferroni correction: psychotherapists scores were greater than nursing staff scores and administrative personnel scores.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Evaluation Outcomes, System Usability Scale</title>
        <p>There were no missing items; therefore, no imputation of values was necessary. The SUS scores were normally distributed with few outliers. The mean SUS global score was 81.49 (SD 11.10). The mean score for the VRAT-CT was 82.00 (SD 12.55), and the mean for the VRAT-CC 80.99 (SD 12.67). Male participants scored slightly higher (mean 81.71, SD 15.24) than female participants (mean 81.39, SD 11.19), but this difference was not statistically significant (<italic>P</italic>=.11). Among the professional groups, psychotherapists (mean 86.44, SD 8.79) scored higher than nursing staff (mean 79.01, SD 13.30) and administrative personnel (mean 77.98, SD 10.72). The difference between psychotherapists and other professional groups reached statistical significance (<italic>F</italic><sub>2,106</sub>=6.136; <italic>P=</italic>.003) (<xref ref-type="table" rid="table1">Table 1</xref> and <xref rid="figure2" ref-type="fig">Figure 2</xref>).</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>System Usability Scale (SUS) scores. Psychotherapists scored significantly higher than nursing staff and administrative personnel (<italic>P</italic>=.003).</p>
          </caption>
          <graphic xlink:href="games_v9i2e26820_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Multilevel Factorial Analysis</title>
        <p>The System Usability Scale produced a Cronbach α=.80 with good correlation between single items. Item loadings ranged from 0.2 to 0.8 (chi-square <italic>P</italic>&#60;.001; comparative fit index 0.905; sample-size adjusted Bayesian information criterion 5028.08; root mean square error of approximation 0.075) and demonstrated different factor structures for each profession (<xref ref-type="table" rid="table2">Table 2</xref>).</p>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Mean score on the System Usability Scale and loadings for each item.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="90"/>
            <col width="100"/>
            <col width="130"/>
            <col width="0"/>
            <col width="100"/>
            <col width="130"/>
            <col width="0"/>
            <col width="100"/>
            <col width="130"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="90"/>
            <thead>
              <tr valign="top">
                <td>Item</td>
                <td colspan="9">Profession</td>
                <td colspan="3">Statistics</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="3">Psychotherapists</td>
                <td colspan="3">Nursing staff</td>
                <td colspan="3">Administrative personnel</td>
                <td colspan="2">
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Loadings</td>
                <td>Mean (SD)</td>
                <td colspan="2">Loadings</td>
                <td>Mean (SD)</td>
                <td colspan="2">Loadings</td>
                <td>Mean (SD)</td>
                <td colspan="2"><italic>F</italic> test (<italic>df1,df2</italic>)</td>
                <td colspan="2"><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1</td>
                <td>0.489</td>
                <td>4.42 (0.67)<sup>b</sup></td>
                <td colspan="2">0.356</td>
                <td>4.16 (0.79)</td>
                <td colspan="2">0.396</td>
                <td>3.92 (1.04)</td>
                <td colspan="2">6.048 (2, 215)</td>
                <td colspan="2">.003</td>
              </tr>
              <tr valign="top">
                <td>2</td>
                <td>0.591</td>
                <td>1.25 (0.46)<sup>b</sup></td>
                <td colspan="2">0.670</td>
                <td>1.35 (0.59)<sup>c</sup></td>
                <td colspan="2">0.687</td>
                <td>1.60 (0.66)</td>
                <td colspan="2">5.993 (2, 215)</td>
                <td colspan="2">.003</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>0.672</td>
                <td>4.45 (0.57)</td>
                <td colspan="2">0.665</td>
                <td>4.28 (1.01)</td>
                <td colspan="2">0.565</td>
                <td>4.33 (0.86)</td>
                <td colspan="2">1.929 (2, 205)</td>
                <td colspan="2">.15</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>0.520</td>
                <td>2.00 (1.07)<sup>a,b</sup></td>
                <td colspan="2">0.428</td>
                <td>2.49 (1.26)</td>
                <td colspan="2">0.328</td>
                <td>2.81 (1.28)</td>
                <td colspan="2">97.663 (2, 215)</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>0.500</td>
                <td>4.44 (0.65)</td>
                <td colspan="2">0.405</td>
                <td>4.31 (0.58)</td>
                <td colspan="2">0.575</td>
                <td>4.38 (0.53)</td>
                <td colspan="2">0.896 (2, 215)</td>
                <td colspan="2">.41</td>
              </tr>
              <tr valign="top">
                <td>6</td>
                <td>0.523</td>
                <td>1.30 (0.54)<sup>a,b</sup></td>
                <td colspan="2">0.513</td>
                <td>1.73 (0.90)</td>
                <td colspan="2">0.579</td>
                <td>1.79 (0.64)</td>
                <td colspan="2">10.070 (2, 215)</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>7</td>
                <td>0.335</td>
                <td>4.31 (0.76)</td>
                <td colspan="2">0.805</td>
                <td>4.24 (0.85)</td>
                <td colspan="2">0.554</td>
                <td>4.08 (0.62)</td>
                <td colspan="2">1.351 (2, 215)</td>
                <td colspan="2">.22</td>
              </tr>
              <tr valign="top">
                <td>8</td>
                <td>0.501</td>
                <td>1.30 (0.58)<sup>a</sup></td>
                <td colspan="2">0.506</td>
                <td>1.71 (1.13)</td>
                <td colspan="2">0.793</td>
                <td>1.40 (0.57)</td>
                <td colspan="2">4.946 (2, 215)</td>
                <td colspan="2">.008</td>
              </tr>
              <tr valign="top">
                <td>9</td>
                <td>0.561</td>
                <td>4.22 (0.73)<sup>b</sup></td>
                <td colspan="2">0.523</td>
                <td>3.90 (1.01)</td>
                <td colspan="2">0.5552</td>
                <td>3.81 (0.97)</td>
                <td colspan="2">4.219 (2, 215)</td>
                <td colspan="2">.02</td>
              </tr>
              <tr valign="top">
                <td>10</td>
                <td>0.291</td>
                <td>1.50 (0.78)<sup>b</sup></td>
                <td colspan="2">0.566</td>
                <td>2.02 (1.13)<sup>c</sup></td>
                <td colspan="2">0.535</td>
                <td>1.73 (0.79)</td>
                <td colspan="2">6.519 (2, 215)</td>
                <td colspan="2">.002</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table2fn1">
              <p><sup>a</sup>Posthoc analysis with Bonferroni correction: psychotherapists scores were greater than nursing staff scores and administrative personnel scores.</p>
            </fn>
            <fn id="table2fn2">
              <p><sup>b</sup>Posthoc analysis with Bonferroni correction: psychotherapists scores were greater than administrative personnel scores.</p>
            </fn>
            <fn id="table2fn3">
              <p><sup>c</sup>Posthoc analysis with Bonferroni correction: nursing staff scores were greater than administrative personnel scores.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>The usability of a digital psychotherapeutic tool developed using a game engine was studied in different professions. The SUS score obtained for the virtual reality human–human interface achieved the benchmark for an excellent system [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>], scoring highest among the professional target group. The sample's demographic characteristics did not affect these results: the SUS scores were similar regardless of age or gender. To the best of our knowledge, this is the first study to assess the usability of a psychotherapeutic VR treatment tool for people experiencing acoustic verbal hallucinations.</p>
      <p>Our study's main strengths are the large sample size and the naturalistic design, particularly the personalized introduction and the practical hands-on approach to the system [<xref ref-type="bibr" rid="ref14">14</xref>,<xref ref-type="bibr" rid="ref15">15</xref>]. We chose this approach to emulate the introduction and instruction used by psychotherapists in research and clinical practice. Furthermore, through the personalized introduction to the system, we sought to compensate for differences in the theoretical backgrounds of the professional groups.</p>
      <p>The SUS was originally developed to evaluate the usability of products and services, including hardware, software, mobile devices, websites, and apps. Because the product was a combination of these elements and has previously been used to test medical devices and products [<xref ref-type="bibr" rid="ref23">23</xref>-<xref ref-type="bibr" rid="ref25">25</xref>], we selected the SUS to enable easy comparison with both similar and dissimilar products or devices [<xref ref-type="bibr" rid="ref21">21</xref>]. The digital therapeutic system tested yielded a score ranging from good to excellent, depending on professional background [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>].</p>
      <p>The virtual reality human–human interface achieved a higher score among professionals with psychotherapeutic backgrounds (ie, psychiatrists and psychologists). Since all participants were naïve to the system, differences cannot be attributed to user experience [<xref ref-type="bibr" rid="ref26">26</xref>]. In our opinion, these differences underscore the need for relevant training and professional background in order to fully understand and use the virtual reality therapeutic tool that we have created [<xref ref-type="bibr" rid="ref14">14</xref>]. Posthoc analysis revealed no differences between psychiatrists and psychologists. We, therefore, consider both to form a uniform group with psychotherapeutic training as the common factor (in Switzerland) [<xref ref-type="bibr" rid="ref27">27</xref>-<xref ref-type="bibr" rid="ref29">29</xref>]. In addition, the similarities between psychologists and psychiatrists, regarding educational level and awareness of relevant research, should also be taken into account [<xref ref-type="bibr" rid="ref30">30</xref>].</p>
      <p>The SUS scale was designed as a global measure of perceived usability. Attempts thus far to identify an underlying factor analysis have been misleading and mainly reflected its alternating structure [<xref ref-type="bibr" rid="ref31">31</xref>]. Nonetheless, we analyzed the SUS at an item level to discern differences potentially attributable to the skill-grade mix of the participants. The SUS scale yielded similar scores between psychotherapists and nonpsychotherapists only for items 3, 5, and 7. These items are more closely related to the handling of the system than to its actual implementation and use in research and clinical practice. The SUS scale also has a different factor structure for each profession, indicating different evaluation patterns for the usability of the system. This leads us to believe that the system is generally easy to use, allowing therapists to quickly become familiar with it and develop confidence, thereby increasing the likelihood of incorporating this system into their therapeutic repertoire and using it to deliver therapy [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref32">32</xref>].</p>
      <p>Our study has several limitations that must be acknowledged. First, we did not include a clinical population. Although people experiencing verbal acoustic hallucinations were involved in the development process [<xref ref-type="bibr" rid="ref16">16</xref>], they were not systematically involved in evaluating the system. At this stage, we chose to focus on the therapeutic end user since they would be responsible for introducing and guiding patients through the system and conducting the therapy sessions afterward. Another factor in our study was the use of only a single session for evaluation. This approach was chosen with the intention of assessing the intuitive usability of the system and to avoid learning effects. We did not compare our therapeutic system with those based on other technical possibilities, such as a non-VR presentation of the Avatar or the use of mobile or handheld devices. It is possible that such technical alternatives may yield a higher usability score. Finally, although no discomfort or side effects were reported, we did not systematically assess these important issues related to the use of VR technology [<xref ref-type="bibr" rid="ref16">16</xref>].</p>
      <p>We were able to demonstrate that a virtual reality human–human interface for research and clinical practice can be developed using an existing and widely available game engine. The results show that the usability of the digital therapeutic tools depends not only on the system itself but also on the user's professional background. We believe this system may enable and encourage psychotherapists to expand their therapeutic skills, to routinely using this technology in research and clinical practice [<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref33">33</xref>-<xref ref-type="bibr" rid="ref35">35</xref>]. In summary, given the high usability scores, gaming technology and platforms seem to be suitable for the creation and customization of novel therapeutic approaches in psychiatry.</p>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Virtual reality human–human interface.</p>
        <media xlink:href="games_v9i2e26820_app1.mp4" xlink:title="MP4 File  (MP4 Video), 41377 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">SUS</term>
          <def>
            <p>System Usability Scale</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">VR</term>
          <def>
            <p>virtual reality</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">VRAT</term>
          <def>
            <p>Virtual Reality Avatar-Creation Tool</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>We thank Mrs Lorna McBroom for manuscript proofreading and language editing.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lazar</surname>
              <given-names>SG</given-names>
            </name>
          </person-group>
          <article-title>The cost-effectiveness of psychotherapy for the major psychiatric diagnoses</article-title>
          <source>Psychodyn Psychiatry</source>
          <year>2014</year>
          <month>09</month>
          <volume>42</volume>
          <issue>3</issue>
          <fpage>423</fpage>
          <lpage>57</lpage>
          <pub-id pub-id-type="doi">10.1521/pdps.2014.42.3.423</pub-id>
          <pub-id pub-id-type="medline">25211432</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schramm</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Gerardi</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rothbaum</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Berger</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Role of psychotherapy in the management of psychiatric diseases</article-title>
          <source>Handb Clin Neurol</source>
          <year>2012</year>
          <volume>106</volume>
          <fpage>633</fpage>
          <lpage>41</lpage>
          <pub-id pub-id-type="doi">10.1016/B978-0-444-52002-9.00037-1</pub-id>
          <pub-id pub-id-type="medline">22608648</pub-id>
          <pub-id pub-id-type="pii">B978-0-444-52002-9.00037-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Craighead</surname>
              <given-names>WE</given-names>
            </name>
            <name name-style="western">
              <surname>Craighead</surname>
              <given-names>LW</given-names>
            </name>
          </person-group>
          <article-title>The role of psychotherapy in treating psychiatric disorders</article-title>
          <source>Med Clin North Am</source>
          <year>2001</year>
          <month>5</month>
          <volume>85</volume>
          <issue>3</issue>
          <fpage>617</fpage>
          <lpage>629</lpage>
          <pub-id pub-id-type="doi">10.1016/s0025-7125(05)70332-1</pub-id>
          <pub-id pub-id-type="medline">11349476</pub-id>
          <pub-id pub-id-type="pii">S0025-7125(05)70332-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bighelli</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Huhn</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schneider-Thoma</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Krause</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Reitmeir</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wallis</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Schwermann</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Pitschel-Walz</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Barbui</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Furukawa</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Leucht</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Response rates in patients with schizophrenia and positive symptoms receiving cognitive behavioural therapy: a systematic review and single-group meta-analysis</article-title>
          <source>BMC Psychiatry</source>
          <year>2018</year>
          <month>12</month>
          <day>04</day>
          <volume>18</volume>
          <issue>1</issue>
          <fpage>380</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcpsychiatry.biomedcentral.com/articles/10.1186/s12888-018-1964-8"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12888-018-1964-8</pub-id>
          <pub-id pub-id-type="medline">30514268</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12888-018-1964-8</pub-id>
          <pub-id pub-id-type="pmcid">PMC6280425</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bighelli</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Salanti</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Huhn</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Schneider-Thoma</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Krause</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Reitmeir</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wallis</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Schwermann</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Pitschel-Walz</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Barbui</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Furukawa</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Leucht</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Psychological interventions to reduce positive symptoms in schizophrenia: systematic review and network meta-analysis</article-title>
          <source>World Psychiatry</source>
          <year>2018</year>
          <month>10</month>
          <volume>17</volume>
          <issue>3</issue>
          <fpage>316</fpage>
          <lpage>329</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1002/wps.20577"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/wps.20577</pub-id>
          <pub-id pub-id-type="medline">30192101</pub-id>
          <pub-id pub-id-type="pmcid">PMC6127754</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <collab>NICE</collab>
          </person-group>
          <article-title>Psychosis and schizophrenia in adults: prevention and management</article-title>
          <source>NICE Guidelines CG178</source>
          <year>2014</year>
          <fpage>1</fpage>
          <lpage>46</lpage>
          <pub-id pub-id-type="medline">32207892</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <collab>American Psychiatric Association</collab>
          </person-group>
          <source>The American Psychiatric Association Practice Guideline for the Treatment of Patients With Schizophrenia</source>
          <year>2020</year>
          <publisher-loc>Washington, DC</publisher-loc>
          <publisher-name>American Psychiatric Association Publishing</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Riva</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Virtual reality in psychotherapy: review</article-title>
          <source>Cyberpsychol Behav</source>
          <year>2005</year>
          <month>06</month>
          <volume>8</volume>
          <issue>3</issue>
          <fpage>220</fpage>
          <lpage>30; discussion 231</lpage>
          <pub-id pub-id-type="doi">10.1089/cpb.2005.8.220</pub-id>
          <pub-id pub-id-type="medline">15971972</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rizzo</surname>
              <given-names>AS</given-names>
            </name>
            <name name-style="western">
              <surname>Koenig</surname>
              <given-names>ST</given-names>
            </name>
            <name name-style="western">
              <surname>Talbot</surname>
              <given-names>TB</given-names>
            </name>
          </person-group>
          <article-title>Clinical virtual reality: emerging opportunities for psychiatry</article-title>
          <source>Focus (Am Psychiatr Publ)</source>
          <year>2018</year>
          <month>07</month>
          <volume>16</volume>
          <issue>3</issue>
          <fpage>266</fpage>
          <lpage>278</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/31975921"/>
          </comment>
          <pub-id pub-id-type="doi">10.1176/appi.focus.20180011</pub-id>
          <pub-id pub-id-type="medline">31975921</pub-id>
          <pub-id pub-id-type="pii">FOC_20180011</pub-id>
          <pub-id pub-id-type="pmcid">PMC6493092</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bell</surname>
              <given-names>IH</given-names>
            </name>
            <name name-style="western">
              <surname>Nicholas</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Alvarez-Jimenez</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Valmaggia</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Virtual reality as a clinical tool in mental health research and practice </article-title>
          <source>Dialogues Clin Neurosci</source>
          <year>2020</year>
          <month>06</month>
          <volume>22</volume>
          <issue>2</issue>
          <fpage>169</fpage>
          <lpage>177</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.dialogues-cns.org/contents-22-2/dialoguesclinneurosci-22-169"/>
          </comment>
          <pub-id pub-id-type="doi">10.31887/DCNS.2020.22.2/lvalmaggia</pub-id>
          <pub-id pub-id-type="medline">32699517</pub-id>
          <pub-id pub-id-type="pmcid">PMC7366939</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Leff</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Huckvale</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Arbuthnot</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Leff</surname>
              <given-names>AP</given-names>
            </name>
          </person-group>
          <article-title>Computer-assisted therapy for medication-resistant auditory hallucinations: proof-of-concept study</article-title>
          <source>Br J Psychiatry</source>
          <year>2013</year>
          <month>06</month>
          <volume>202</volume>
          <fpage>428</fpage>
          <lpage>33</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://bjp.rcpsych.org/cgi/pmidlookup?view=long&#38;pmid=23429202"/>
          </comment>
          <pub-id pub-id-type="doi">10.1192/bjp.bp.112.124883</pub-id>
          <pub-id pub-id-type="medline">23429202</pub-id>
          <pub-id pub-id-type="pii">bjp.bp.112.124883</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Craig</surname>
              <given-names>TK</given-names>
            </name>
            <name name-style="western">
              <surname>Rus-Calafell</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Leff</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Huckvale</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Howarth</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Emsley</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Garety</surname>
              <given-names>PA</given-names>
            </name>
          </person-group>
          <article-title>AVATAR therapy for auditory verbal hallucinations in people with psychosis: a single-blind, randomised controlled trial</article-title>
          <source>Lancet Psychiatry</source>
          <year>2018</year>
          <month>12</month>
          <volume>5</volume>
          <issue>1</issue>
          <fpage>31</fpage>
          <lpage>40</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S2215-0366(17)30427-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2215-0366(17)30427-3</pub-id>
          <pub-id pub-id-type="medline">29175276</pub-id>
          <pub-id pub-id-type="pii">S2215-0366(17)30427-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC5746597</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mishkind</surname>
              <given-names>MC</given-names>
            </name>
            <name name-style="western">
              <surname>Norr</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Katz</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Reger</surname>
              <given-names>GM</given-names>
            </name>
          </person-group>
          <article-title>Review of virtual reality treatment in psychiatry: evidence versus current diffusion and use</article-title>
          <source>Curr Psychiatry Rep</source>
          <year>2017</year>
          <month>9</month>
          <day>18</day>
          <volume>19</volume>
          <issue>80</issue>
          <fpage>1</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1007/s11920-017-0836-0</pub-id>
          <pub-id pub-id-type="medline">28920179</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rus-Calafell</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Garety</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Sason</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Craig</surname>
              <given-names>TJK</given-names>
            </name>
            <name name-style="western">
              <surname>Valmaggia</surname>
              <given-names>LR</given-names>
            </name>
          </person-group>
          <article-title>Virtual reality in the assessment and treatment of psychosis: a systematic review of its utility, acceptability and effectiveness</article-title>
          <source>Psychol Med</source>
          <year>2018</year>
          <month>02</month>
          <volume>48</volume>
          <issue>3</issue>
          <fpage>362</fpage>
          <lpage>391</lpage>
          <pub-id pub-id-type="doi">10.1017/S0033291717001945</pub-id>
          <pub-id pub-id-type="medline">28735593</pub-id>
          <pub-id pub-id-type="pii">S0033291717001945</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schwartzman</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Segal</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Drapeau</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Perceptions of virtual reality among therapists who do not apply this technology in clinical practice</article-title>
          <source>Psychol Serv</source>
          <year>2012</year>
          <month>08</month>
          <volume>9</volume>
          <issue>3</issue>
          <fpage>310</fpage>
          <lpage>5</lpage>
          <pub-id pub-id-type="doi">10.1037/a0026801</pub-id>
          <pub-id pub-id-type="medline">22867123</pub-id>
          <pub-id pub-id-type="pii">2012-20935-004</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Birckhead</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Khalil</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Conovitz</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Rizzo</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Danovitch</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Bullock</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Spiegel</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Recommendations for methodology of virtual reality clinical trials in health care by an international working group: iterative study</article-title>
          <source>JMIR Ment Health</source>
          <year>2019</year>
          <month>01</month>
          <day>31</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>e11973</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mental.jmir.org/2019/1/e11973/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11973</pub-id>
          <pub-id pub-id-type="medline">30702436</pub-id>
          <pub-id pub-id-type="pii">v6i1e11973</pub-id>
          <pub-id pub-id-type="pmcid">PMC6374734</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Leff</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Huckvale</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Arbuthnot</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Leff</surname>
              <given-names>AP</given-names>
            </name>
          </person-group>
          <article-title>Avatar therapy for persecutory auditory hallucinations: what is it and how does it work?</article-title>
          <source>Psychosis</source>
          <year>2014</year>
          <month>06</month>
          <volume>6</volume>
          <issue>2</issue>
          <fpage>166</fpage>
          <lpage>176</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24999369"/>
          </comment>
          <pub-id pub-id-type="doi">10.1080/17522439.2013.773457</pub-id>
          <pub-id pub-id-type="medline">24999369</pub-id>
          <pub-id pub-id-type="pmcid">PMC4066885</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>du Sert</surname>
              <given-names>OP</given-names>
            </name>
            <name name-style="western">
              <surname>Potvin</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lipp</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Dellazizzo</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Laurelli</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Breton</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lalonde</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Phraxayavong</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>O'Connor</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Pelletier</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Boukhalfi</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Renaud</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Dumais</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Virtual reality therapy for refractory auditory verbal hallucinations in schizophrenia: a pilot clinical trial</article-title>
          <source>Schizophr Res</source>
          <year>2018</year>
          <month>02</month>
          <day>24</day>
          <fpage>176</fpage>
          <lpage>181</lpage>
          <pub-id pub-id-type="doi">10.1016/j.schres.2018.02.031</pub-id>
          <pub-id pub-id-type="medline">29486956</pub-id>
          <pub-id pub-id-type="pii">S0920-9964(18)30108-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bonfiglio</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>DeepMind partners with gaming company for AI research</article-title>
          <source>The Daily Dot</source>
          <year>2018</year>
          <access-date>2020-11-20</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.dailydot.com/debug/unity-deempind-ai/">https://www.dailydot.com/debug/unity-deempind-ai/</ext-link>
            <ext-link ext-link-type="webcite" xlink:href="https://www.dailydot.com/debug/unity-deempind-ai/"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brooke</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>SUS: a 'quick and dirty' usability scale</article-title>
          <source>Usability Evaluation in Industry</source>
          <year>1996</year>
          <publisher-loc>Boca Raton, Florida</publisher-loc>
          <publisher-name>CRC Press</publisher-name>
          <fpage>189</fpage>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kortum</surname>
              <given-names>PT</given-names>
            </name>
            <name name-style="western">
              <surname>Bangor</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Usability ratings for everyday products measured with the System Usability Scale</article-title>
          <source>Int J Hum Comput Interact</source>
          <year>2013</year>
          <month>01</month>
          <volume>29</volume>
          <issue>2</issue>
          <fpage>67</fpage>
          <lpage>76</lpage>
          <pub-id pub-id-type="doi">10.1080/10447318.2012.681221</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lewis</surname>
              <given-names>JR</given-names>
            </name>
            <name name-style="western">
              <surname>Sauro</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Item benchmarks for the system usability scale</article-title>
          <source>J Usability Stud</source>
          <year>2018</year>
          <volume>13</volume>
          <issue>3</issue>
          <fpage>158</fpage>
          <lpage>167</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Maramba</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Chatterjee</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Newman</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Methods of usability testing in the development of eHealth applications: a scoping review</article-title>
          <source>Int J Med Inform</source>
          <year>2019</year>
          <month>12</month>
          <volume>126</volume>
          <fpage>95</fpage>
          <lpage>104</lpage>
          <pub-id pub-id-type="doi">10.1016/j.ijmedinf.2019.03.018</pub-id>
          <pub-id pub-id-type="medline">31029270</pub-id>
          <pub-id pub-id-type="pii">S1386-5056(18)31318-2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Booth</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Jean-Louis</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Yeung</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gratzer</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Strudwick</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>A primer on usability assessment approaches for health-related applications of virtual reality</article-title>
          <source>JMIR Serious Games</source>
          <year>2020</year>
          <month>10</month>
          <day>28</day>
          <volume>8</volume>
          <issue>4</issue>
          <fpage>e18153</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://games.jmir.org/2020/4/e18153/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/18153</pub-id>
          <pub-id pub-id-type="medline">33112243</pub-id>
          <pub-id pub-id-type="pii">v8i4e18153</pub-id>
          <pub-id pub-id-type="pmcid">PMC7657728</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zapata</surname>
              <given-names>BC</given-names>
            </name>
            <name name-style="western">
              <surname>Fernández-Alemán</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Idri</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Toval</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Empirical studies on usability of mHealth apps: a systematic literature review</article-title>
          <source>J Med Syst</source>
          <year>2015</year>
          <month>02</month>
          <volume>39</volume>
          <issue>2</issue>
          <fpage>1</fpage>
          <lpage>19</lpage>
          <pub-id pub-id-type="doi">10.1007/s10916-014-0182-2</pub-id>
          <pub-id pub-id-type="medline">25600193</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McLellan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Muddimer</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Peres</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The effect of experience on System Usability Scale ratings</article-title>
          <source>J Usability Stud</source>
          <year>2012</year>
          <volume>7</volume>
          <issue>2</issue>
          <fpage>56</fpage>
          <lpage>67</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Caspar</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>The current status of psychotherapy integration in Germany and Switzerland</article-title>
          <source>J Psychother Integr</source>
          <year>2008</year>
          <volume>18</volume>
          <issue>1</issue>
          <fpage>74</fpage>
          <lpage>78</lpage>
          <pub-id pub-id-type="doi">10.1037/1053-0479.18.1.74</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rubo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Martin-Soelch</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Munsch</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Education and training in clinical psychology and psychological psychotherapy in Switzerland</article-title>
          <source>Clin Psychol Eur</source>
          <year>2020</year>
          <month>09</month>
          <day>30</day>
          <volume>2</volume>
          <issue>3</issue>
          <fpage>1</fpage>
          <lpage>11</lpage>
          <pub-id pub-id-type="doi">10.32872/cpe.v2i3.2991</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Oakley</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Malik</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Psychiatric training in Europe</article-title>
          <source>Psychiatrist</source>
          <year>2018</year>
          <month>01</month>
          <day>02</day>
          <volume>34</volume>
          <issue>10</issue>
          <fpage>447</fpage>
          <lpage>450</lpage>
          <pub-id pub-id-type="doi">10.1192/pb.bp.109.026062</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Swanson</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Walker</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Academic versus non-academic emerging adult college student technology use</article-title>
          <source>Tech Know Learn</source>
          <year>2015</year>
          <month>5</month>
          <day>7</day>
          <volume>20</volume>
          <issue>2</issue>
          <fpage>147</fpage>
          <lpage>158</lpage>
          <pub-id pub-id-type="doi">10.1007/s10758-015-9258-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lewis</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sauro</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Revisiting the factor structure of the System Usability Scale</article-title>
          <source>J Usability Stud</source>
          <year>2017</year>
          <volume>12</volume>
          <issue>4</issue>
          <fpage>183</fpage>
          <lpage>192</lpage>
          <pub-id pub-id-type="doi">10.1007/978-3-642-02806-9_12</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Spurgeon</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Wright</surname>
              <given-names>JH</given-names>
            </name>
          </person-group>
          <article-title>Computer-assisted cognitive-behavioral therapy</article-title>
          <source>Curr Psychiatry Rep</source>
          <year>2010</year>
          <month>12</month>
          <volume>12</volume>
          <issue>6</issue>
          <fpage>547</fpage>
          <lpage>552</lpage>
          <pub-id pub-id-type="doi">10.1007/s11920-010-0152-4</pub-id>
          <pub-id pub-id-type="medline">20872100</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Segal</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Bhatia</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Drapeau</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Therapists' perception of benefits and costs of using virtual reality treatments</article-title>
          <source>Cyberpsychol Behav Soc Netw</source>
          <year>2011</year>
          <volume>14</volume>
          <issue>1-2</issue>
          <fpage>29</fpage>
          <lpage>34</lpage>
          <pub-id pub-id-type="doi">10.1089/cyber.2009.0398</pub-id>
          <pub-id pub-id-type="medline">21329440</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wampold</surname>
              <given-names>BE</given-names>
            </name>
          </person-group>
          <article-title>How important are the common factors in psychotherapy? an update</article-title>
          <source>World Psychiatry</source>
          <year>2015</year>
          <month>10</month>
          <volume>14</volume>
          <issue>3</issue>
          <fpage>270</fpage>
          <lpage>7</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://dx.doi.org/10.1002/wps.20238"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/wps.20238</pub-id>
          <pub-id pub-id-type="medline">26407772</pub-id>
          <pub-id pub-id-type="pmcid">PMC4592639</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Spek</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Nyklícek</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Riper</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Keyzer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Pop</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Internet-based cognitive behaviour therapy for symptoms of depression and anxiety: a meta-analysis</article-title>
          <source>Psychol Med</source>
          <year>2006</year>
          <month>11</month>
          <day>20</day>
          <volume>37</volume>
          <issue>03</issue>
          <fpage>319</fpage>
          <lpage>328</lpage>
          <pub-id pub-id-type="doi">10.1017/s0033291706008944</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
