<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
    <front>
        <journal-meta>
            <journal-id journal-id-type="publisher-id">JSG</journal-id>
            <journal-id journal-id-type="nlm-ta">JMIR Serious Games</journal-id>
            <journal-title>JMIR Serious Games</journal-title>
            <issn pub-type="epub">2291-9279</issn>
            <publisher>
                <publisher-name>JMIR Publications Inc.</publisher-name>
                <publisher-loc>Toronto, Canada</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="publisher-id">v2i2e11</article-id>
            <article-id pub-id-type="pmid">25654163</article-id>
            <article-id pub-id-type="doi">10.2196/games.3825</article-id>
            <article-categories>
                <subj-group subj-group-type="heading">
                    <subject>Viewpoint</subject>
                </subj-group>
                <subj-group subj-group-type="article-type">
                    <subject>Viewpoint</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>How to Systematically Assess Serious Games Applied to Health Care</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="editor">
                    <name>
                        <surname>Eysenbach</surname>
                        <given-names>Gunther</given-names>
                    </name>
                </contrib>
            </contrib-group>
            <contrib-group>
                <contrib contrib-type="reviewer">
                    <name>
                        <surname>Paraskevopoulos</surname>
                        <given-names>Ioannis</given-names>
                    </name>
                </contrib>
            </contrib-group>
            <contrib-group>
                <contrib contrib-type="author" id="contrib1">
                    <name name-style="western">
                        <surname>Graafland</surname>
                        <given-names>Maurits</given-names>
                    </name>
                    <degrees>MD</degrees>
                    <xref rid="aff1" ref-type="aff">1</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-1059-698X</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib2">
                    <name name-style="western">
                        <surname>Dankbaar</surname>
                        <given-names>Mary</given-names>
                    </name>
                    <degrees>MSc</degrees>
                    <xref rid="aff2" ref-type="aff">2</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-0880-505X</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib3">
                    <name name-style="western">
                        <surname>Mert</surname>
                        <given-names>Agali</given-names>
                    </name>
                    <degrees>MD, PhD</degrees>
                    <xref rid="aff3" ref-type="aff">3</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-3419-0407</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib4">
                    <name name-style="western">
                        <surname>Lagro</surname>
                        <given-names>Joep</given-names>
                    </name>
                    <degrees>MD, PhD</degrees>
                    <xref rid="aff4" ref-type="aff">4</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-0787-9498</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib5">
                    <name name-style="western">
                        <surname>De Wit-Zuurendonk</surname>
                        <given-names>Laura</given-names>
                    </name>
                    <degrees>MD</degrees>
                    <xref rid="aff5" ref-type="aff">5</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-4302-4534</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib6">
                    <name name-style="western">
                        <surname>Schuit</surname>
                        <given-names>Stephanie</given-names>
                    </name>
                    <degrees>MD, PhD</degrees>
                    <xref rid="aff6" ref-type="aff">6</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-1200-9961</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib7">
                    <name name-style="western">
                        <surname>Schaafstal</surname>
                        <given-names>Alma</given-names>
                    </name>
                    <degrees>PhD</degrees>
                    <xref rid="aff7" ref-type="aff">7</xref>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-0832-2827</ext-link>
                </contrib>
                <contrib contrib-type="author" id="contrib8" corresp="yes">
                    <name name-style="western">
                        <surname>Schijven</surname>
                        <given-names>Marlies</given-names>
                    </name>
                    <degrees>MD, PhD, MHSc</degrees>
                    <xref rid="aff1" ref-type="aff">1</xref>
                    <address>
                        <institution>Department of Surgery</institution>
                        <institution>Academic Medical Center</institution>
                        <addr-line>PO Box 22660</addr-line>
                        <addr-line>Amsterdam, 1100DD</addr-line>
                        <country>Netherlands</country>
                        <phone>31 20 566 4207</phone>
                        <fax>31 20 691 4858</fax>
                        <email>m.p.schijven@amc.uva.nl</email>
                    </address>
                    <ext-link ext-link-type="orcid">http://orcid.org/0000-0001-7013-0116</ext-link>
                </contrib>
            </contrib-group>
            <aff id="aff1">
                <sup>1</sup>
                <institution>Department of Surgery</institution>
                <institution>Academic Medical Center</institution>
                <addr-line>Amsterdam</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff2">
                <sup>2</sup>
                <institution>Desiderius School</institution>
                <institution>Department of Medical Education</institution>
                <institution>Erasmus University Medical Centre</institution>
                <addr-line>Rotterdam</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff3">
                <sup>3</sup>
                <institution>National Military Rehabilitation Centre</institution>
                <addr-line>Doorn</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff4">
                <sup>4</sup>
                <institution>Department of Geriatrics</institution>
                <institution>Haga Ziekenhuis</institution>
                <addr-line>Den Haag</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff5">
                <sup>5</sup>
                <institution>Department of Obstetrics and Gynaecology</institution>
                <institution>M&#225;xima Medical Centre</institution>
                <addr-line>Veldhoven</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff6">
                <sup>6</sup>
                <institution>Departments of Emergency Medicine and Internal Medicine</institution>
                <institution>Erasmus University Medical Centre</institution>
                <addr-line>Rotterdam</addr-line>
                <country>Netherlands</country>
            </aff>
            <aff id="aff7">
                <sup>7</sup>
                <institution>Department of ICT</institution>
                <institution>Windesheim University of Applied Sciences</institution>
                <addr-line>Zwolle</addr-line>
                <country>Netherlands</country>
            </aff>
            <author-notes>
                <corresp>Corresponding Author: Marlies Schijven <email>m.p.schijven@amc.uva.nl</email>
                </corresp>
            </author-notes>
            <pub-date pub-type="collection">
                <season>Jul-Dec</season>
                <year>2014</year>
            </pub-date>
            <pub-date pub-type="epub">
                <day>11</day>
                <month>11</month>
                <year>2014</year>
            </pub-date>
            <volume>2</volume>
            <issue>2</issue>
            <elocation-id>e11</elocation-id>
            <!--history from ojs - api-xml-->
            <history>
                <date date-type="received">
                    <day>31</day>
                    <month>08</month>
                    <year>2014</year>
                </date>
                <date date-type="rev-request">
                    <day>10</day>
                    <month>09</month>
                    <year>2014</year>
                </date>
                <date date-type="rev-recd">
                    <day>15</day>
                    <month>10</month>
                    <year>2014</year>
                </date>
                <date date-type="accepted">
                    <day>17</day>
                    <month>10</month>
                    <year>2014</year>
                </date>
            </history>
            <!--(c) the authors - correct author names and publication date here if necessary. Date in form ', dd.mm.yyyy' after jmir.org-->
            <copyright-statement>&#169;Maurits Graafland, Mary Dankbaar, Agali Mert, Joep Lagro, Laura De Wit-Zuurendonk, Stephanie Schuit, Alma Schaafstal, Marlies Schijven. Originally published in JMIR Serious Games (http://games.jmir.org), 11.11.2014. </copyright-statement>
            <copyright-year>2014</copyright-year>
            <license license-type="open-access" xlink:href="http://creativecommons.org/licenses/by/2.0/">
                <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (http://creativecommons.org/licenses/by/2.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Serious Games, is properly cited. The complete bibliographic information, a link to the original publication on http://games.jmir.org, as well as this copyright and license information must be included.</p>
            </license>
            <self-uri xlink:href="http://games.jmir.org/2014/2/e11/" xlink:type="simple" />
            <abstract>
                <p>The usefulness and effectiveness of specific serious games in the medical
domain is often unclear. This is caused by a lack of supporting
evidence on validity of individual games, as well as a lack of publicly
available information. Moreover, insufficient understanding of design
principles among the individuals and institutions that develop or apply a medical
serious game compromises their use. This article provides the first
consensus-based framework for the assessment of specific medical serious
games. The framework provides 62 items in 5 main themes, aimed at assessing
a serious game&#8217;s rationale, functionality, validity, and data safety. This
will allow caregivers and educators to make balanced choices when applying a
serious game for healthcare purposes. Furthermore, the framework provides
game manufacturers with standards for the development of new, valid serious
games.</p>
            </abstract>
            <kwd-group>
                <kwd>consensus</kwd>
                <kwd>serious game</kwd>
                <kwd>applied game</kwd>
                <kwd>telehealth</kwd>
                <kwd>mobile health</kwd>
                <kwd>video game</kwd>
            </kwd-group>
        </article-meta>
    </front>
    <body>
        <sec>
            <title>Background</title>
            <p>
                <italic>Serious</italic> or <italic>applied games</italic> are digital games with the purpose to improve individual&#8217;s knowledge, skills, or attitudes in the &#8220;real&#8221; world. Serious games applied to medical or health-related purposes are growing rapidly in numbers and in types of applications. Serious games have been shown to be at least as effective as conventional tests in improving cognitive abilities in the elderly [<xref ref-type="bibr" rid="ref1">1</xref>] and even more effective than conventional neuropsychological interventions when it comes to improving neuropsychological abilities of alcoholic patients [<xref ref-type="bibr" rid="ref2">2</xref>]. Serious game-based interventions have been used to support rehabilitation in disabled patients, showing equal effectiveness compared to conventional training programs [<xref ref-type="bibr" rid="ref3">3</xref>]. Games have been applied to promote healthy behavior in children [<xref ref-type="bibr" rid="ref4">4</xref>] and educate patients [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref6">6</xref>]. Serious game-based patient education has also been shown to increase the treatment adherence among adolescents with leukemia [<xref ref-type="bibr" rid="ref7">7</xref>]. A third application for serious games is training medical personnel [<xref ref-type="bibr" rid="ref8">8</xref>]. Serious games have been shown to add to Advanced Life Support training [<xref ref-type="bibr" rid="ref9">9</xref>] and improve understanding of geriatrics principles among medical students compared to conventional training methods [<xref ref-type="bibr" rid="ref10">10</xref>]. Patients [<xref ref-type="bibr" rid="ref11">11</xref>], students, and professionals [<xref ref-type="bibr" rid="ref12">12</xref>] generally view game-based interventions as fun and challenging.</p>
            <p>Although results for serious games in terms of effectiveness for such purposes are promising, their implementation as &#8220;serious&#8221; modalities for prevention, treatment, or training in health care is hindered by lack of understanding of the underlying concepts among health care professionals, or even distrust. Before doctors and patients consider using serious games as a useful solution for a health care-related problem, it is important that they understand what problem is being addressed by the game and that a proposed claim on effectiveness is indeed trustworthy. Many clinicians are currently undereducated in judging a serious game&#8217;s safety or effectiveness. Information on individual games is often hard to find in disorganized app stores and websites [<xref ref-type="bibr" rid="ref13">13</xref>]. Studies on serious games&#8217; validity and effectiveness remain scarce [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref14">14</xref>]. The idea of applying a video game in health care may even be resentful to certain clinicians or patients. In addition, threats to data safety fuel distrust towards electronic applications in health care altogether [<xref ref-type="bibr" rid="ref15">15</xref>]. Such issues menace the practical application of serious games throughout health care, subsequently limiting investments in smart solutions that may actually prove beneficial in the end.</p>
            <p>This article discusses the first tool for the systematic assessment of serious games applied to medical use, for educators and clinicians. The information collected and organized accordingly, will aid health care practitioners to understand and appraise the risks and benefits of specific serious games in health care in a uniform manner.</p>
        </sec>
        <sec>
            <title>Assessment Framework</title>
            <p>To our knowledge, there is currently no systematic framework for the assessment of serious games in health care described in literature. Therefore, the Dutch Society for Simulation in Healthcare (DSSH) [<xref ref-type="bibr" rid="ref16">16</xref>] has developed a consensus-based framework, categorizing important items that assess a serious game&#8217;s safety and validity. Eight individuals (see Acknowledgements section for details) from six different institutions experienced in designing, applying, or researching serious games for health care-related purposes participated. The reporting standards for <italic>non-</italic>game mobile health apps for medical purposes (mHealth), published by Lewis [<xref ref-type="bibr" rid="ref17">17</xref>] and Albrecht [<xref ref-type="bibr" rid="ref18">18</xref>], was used as a basis. This system is applied by the peer-reviewed mHealth app assessment initiative of the <italic>Journal of Medical Internet Research</italic> [<xref ref-type="bibr" rid="ref19">19</xref>]. Due to inherent differences in the functionality of <italic>games</italic> compared to purely informational <italic>mHealth</italic> applications, this framework required re-evaluation.</p>
            <p>The panel reviewed the items from these reporting standards during two meetings. All items in the Albrecht framework [<xref ref-type="bibr" rid="ref18">18</xref>] were systematically evaluated. For each of the 5 categories, items irrelevant to serious games were removed and if necessary, extra items were added. During the second panel meeting, the framework was re-evaluated and all members approved the final version.</p>
            <p>The framework described provides 62 items in 5 main themes (<xref ref-type="table" rid="table1">Table 1</xref>), aimed at assessing a serious game&#8217;s rationale, functionality, validity, and data safety. It specifically does <italic>not</italic> aim to assess its effectiveness in terms of success or user attractiveness. The panel defined serious games (other than a regular medical application) as digital applications instigating a specific behavioral change to its user, in the form of skills, knowledge, or attitudes useful to reality [<xref ref-type="bibr" rid="ref20">20</xref>]. The framework does therefore <italic>not</italic> apply to (mobile or Web-based) digital health apps with a purely informational purpose, for which the <italic>mHealth</italic> app assessment framework is designed [<xref ref-type="bibr" rid="ref18">18</xref>].</p>
            <table-wrap position="float" id="table1">
                <label>Table 1</label>
                <caption>
                    <p>Items relevant for the assessment of a serious game used for health care-related purposes.</p>
                </caption>
                <table width="661" border="1" cellpadding="7" cellspacing="0" rules="groups" frame="hsides">
                    <col width="93" />
                    <col width="114" />
                    <col width="122" />
                    <col width="274" />
                    <thead>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Category</td>
                            <td>Item</td>
                            <td>Question</td>
                        </tr>
                    </thead>
                    <tbody>
                        <tr valign="top">
                            <td rowspan="14">Game description</td>
                            <td>Meta-data</td>
                            <td>Operating system</td>
                            <td>Operating systems of the game</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Version</td>
                            <td>Version</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Web-link</td>
                            <td>Web-link</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Project type</td>
                            <td>Commercial, non-commercial, other</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Access</td>
                            <td>Public / restricted / other</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Adjunct devices</td>
                            <td>Is an adjunct device needed?</td>
                        </tr>
                        <tr valign="top">
                            <td>Development</td>
                            <td>Funding</td>
                            <td>How was development funded? Eg, funding agencies, investors</td>
                        </tr>
                        <tr valign="top">
                            <td>Sponsoring / Advertising</td>
                            <td>Advertisement policy</td>
                            <td>Is the game free of commercial pop-ups?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>
                                <break />
                            </td>
                            <td>If not, what is advertised?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Sources of income</td>
                            <td>Are there sources of income within the game?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Sources of income outside game</td>
                            <td>What are the sources of income of the owner/distributor?</td>
                        </tr>
                        <tr valign="top">
                            <td>Potential conflicts of interest</td>
                            <td>Affiliations</td>
                            <td>What affiliations do the publishers have that could influence content or user group?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Conflicts of interest</td>
                            <td>What interests do the publishers have that could influence the game&#8217;s content or user group?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Disclosure</td>
                            <td>Are conflicts of interest disclosed?</td>
                        </tr>
                        <tr valign="top">
                            <td rowspan="12">Rationale</td>
                            <td>Purpose</td>
                            <td>Goal or purpose</td>
                            <td>What is (are) the purpose(s) of the game?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Disclosure</td>
                            <td>Is (are) the purpose(s) disclosed to users?</td>
                        </tr>
                        <tr valign="top">
                            <td>Medical device</td>
                            <td>Medical device</td>
                            <td>Is the serious game a medical device, or not?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Class</td>
                            <td>If yes, which class?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Approval by legal bodies</td>
                            <td>If yes, does it comply to the necessary requirements (FDA-approval, CE-mark?).</td>
                        </tr>
                        <tr valign="top">
                            <td>User group</td>
                            <td>Specific user groups</td>
                            <td>For each user group: disease/condition, or health care profession.</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Description</td>
                            <td>Please specify gender, age (range), and other relevant descriptive items.</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Limits</td>
                            <td>Are there age limits, or other limits?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Disclosure</td>
                            <td>Is the intended user group disclosed?</td>
                        </tr>
                        <tr valign="top">
                            <td>Setting</td>
                            <td>Patient care</td>
                            <td>Is the game used in patient care?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Training courses</td>
                            <td>Is the game used in training courses or -curricula?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>SCORM compliancy</td>
                            <td>If used in training courses or curricula, is the serious game SCORM-compliant?</td>
                        </tr>
                        <tr valign="top">
                            <td rowspan="13">Functionality</td>
                            <td>Purposes / didactic features</td>
                            <td>For every purpose of the game:</td>
                            <td>
                                <break />
                            </td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Learning or behavioral goals</td>
                            <td>What content will the player learn?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Relation learning and gameplay</td>
                            <td>How does the learning content relate to the gameplay?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Instruction</td>
                            <td>What intervention leads to the learning transition (eg, tutorial, instructions (in-game))</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Assessment (progress) in game</td>
                            <td>Through which parameters is progress in the game measured?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Assessment parameters</td>
                            <td>Which parameters are to designers' opinion indicative for measuring learning effects?</td>
                        </tr>
                        <tr valign="top">
                            <td>Content Management</td>
                            <td>Content Management system</td>
                            <td>Is the Content Management System restricted to specified persons or institutions?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>User uploaded content</td>
                            <td>If no, are users allowed to upload their own content?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Content monitoring</td>
                            <td>How is uploaded content checked?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Restrictions and limits of the serious game</td>
                            <td>Please describe restrictions and limits of the serious game. What content on the learning goals is not covered?</td>
                        </tr>
                        <tr valign="top">
                            <td>Potentially undesirable effects</td>
                            <td>Potentially undesirable effects</td>
                            <td>What potential undesirable effects could the game have?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Disclosure</td>
                            <td>Are such potential undesirable effects disclosed to the user?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Measures taken</td>
                            <td>What measures are taken to prevent potential undesirable effects?</td>
                        </tr>
                        <tr valign="top">
                            <td rowspan="10">Validity</td>
                            <td>Design process</td>
                            <td>Medical expert complicity</td>
                            <td>Were medical experts (content experts) involved in the design process from the start?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>User group complicity</td>
                            <td>Were representatives from the user group involved in the design process from the start?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Educationalist complicity</td>
                            <td>Were educationalists involved in the design process from the start?</td>
                        </tr>
                        <tr valign="top">
                            <td>User testing</td>
                            <td>User testing</td>
                            <td>Did user testing take place? What were the results, and how were these incorporated in the design?</td>
                        </tr>
                        <tr valign="top">
                            <td>Stability</td>
                            <td>Platform stability</td>
                            <td>Does the game produce the same results on different platforms?</td>
                        </tr>
                        <tr valign="top">
                            <td>Validity (effectiveness)</td>
                            <td>Face validity</td>
                            <td>Do educators and trainees view it as a valid way of instruction?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Content validity</td>
                            <td>How is its content validated to be complete, correct, and nothing but the intended medical construct?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Construct validity</td>
                            <td>Is the game able to measure differences in skills it intends to measure?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Concurrent validity</td>
                            <td>How does learning outcome compare to other methods assessing the same medical construct?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Predictive validity</td>
                            <td>Does playing the game predict skills improvement in real life?</td>
                        </tr>
                        <tr valign="top">
                            <td rowspan="9">Data protection</td>
                            <td>Data protection and privacy</td>
                            <td>Data processing</td>
                            <td>How is data collected in the serious game?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td rowspan="2">Patient privacy</td>
                            <td>Are patient-specific data stored in the game?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>If yes, are patient informed consent criteria met according to relevant national standards?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Data ownership</td>
                            <td>Who owns and stores the data resulting from play?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Data storage period</td>
                            <td>During what period are data stored?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Data removal</td>
                            <td>Can the user delete data temporarily and/or permanently?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Data storage security</td>
                            <td>Is the data storage secured in conformity with laws of the countries stated above?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Data transmission security</td>
                            <td>Is the data transmission secured in conformity with laws of the countries stated above?</td>
                        </tr>
                        <tr valign="top">
                            <td>
                                <break />
                            </td>
                            <td>Disclosure</td>
                            <td>Are all items on &#8220;data protection&#8221; disclosed to the user?</td>
                        </tr>
                    </tbody>
                </table>
            </table-wrap>
        </sec>
        <sec>
            <title>Assessing Medical Serious Games</title>
            <sec>
                <title>Game Description</title>
                <p>When evaluating a specific serious game, it should be thoroughly described and registered (including information about the manufacturer or owner to whom the game should be attributed and the version). Equally to mobile applications, a special interest is taken into the owner&#8217;s policy concerning revenues from sponsoring and advertisements, both during development as well as its use. Sources of revenue and affiliations (eg, pharmaceutical industry) may bias or threaten a serious game&#8217;s validity for obvious reasons. These should be fully disclosed to the game&#8217;s users. Sources of income within a game can be equally relevant to the costs required for the initial purchase.</p>
            </sec>
            <sec>
                <title>Rationale</title>
                <p>This clarifies the game&#8217;s purpose outside the game. This external purpose (eg, improving eye-hand coordination in laparoscopic surgery) may differ from the actual goal <italic>in</italic> the game (eg, completing a quest in an underground world [<xref ref-type="bibr" rid="ref21">21</xref>] or playing a tennis game [<xref ref-type="bibr" rid="ref22">22</xref>]). This clearly differs from the Albrecht framework, because most mHealth apps have a single obvious purpose (internal goal = external goal). A game&#8217;s purpose relates to the intended user group and the setting in which it is used, similar to mHealth apps.</p>
                <p>Additionally, serious games might fall within the scope of the medical devices, requiring specific guidelines to be implemented, set by the US Food &#38; Drug Administration (FDA), European Committee (Conformit&#233; Europ&#233;enne<italic>,</italic> CE), or national equivalents. This specifically applies to games with a distinct diagnostic or therapeutic purpose. Moreover, integration of serious games into electronic learning environments may demand certain technical requirements. The industry has set standards to improve the interoperability of e-learning content (the Sharable Content Object Reference Model; SCORM)[<xref ref-type="bibr" rid="ref23">23</xref>]. Its implementation will improve the integration of educative serious games in learning management software.</p>
            </sec>
            <sec>
                <title>Functionality</title>
                <p>Functionality of a serious game clearly differs from that of an mHealth app. These usually contain &#8220;dry&#8221; content (eg, medical information) or an obvious functionality (eg, communicating or registering information), whereas a game requires the user to operate or interact with the content, with the ultimate goal to change ones behavior in real life (ie, learning). To understanding this process, information is required on the game&#8217;s content, how the instruction is delivered, how performance is assessed and how these aspects are integrated in the gameplay [<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref25">25</xref>].</p>
                <p>Consequently, it is important to register information on the game&#8217;s content management. For instance, users may be able to add content themselves, making content validation an important issue. This directly influences the game&#8217;s content&#8217;s validity.</p>
                <p>Finally, undesired results or negative transfer of learning could occur in the interaction with a serious game, which is not the same concept as &#8220;gaming the game&#8221; (ie, cheating), an effect that may very well enhance learning [<xref ref-type="bibr" rid="ref24">24</xref>]. If validation research is not present, at least a logical connection between gameplay and behavioral or learning goals should be present and disclosed by the developer.</p>
            </sec>
            <sec>
                <title>Validity</title>
                <p>Validity determines whether an instructional instrument (such as a serious game) adequately resembles the construct it aims to educate or measure<italic>.</italic> More formally, &#8220;the degree to which evidence and theory supports the interpretations of [game] scores entailed by the proposed use of [the game]&#8221; [<xref ref-type="bibr" rid="ref26">26</xref>]. The American Psychological Association has set a series of standards to measure validity [<xref ref-type="bibr" rid="ref26">26</xref>]. Whereas many validity types have been described, validity research in medical education usually contains several consequential phases [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. First, experts should scrutinize the game&#8217;s content to determine its legitimacy (<italic>content</italic> validity). Second, experts and novices judge the instrument&#8217;s apparent similarity to the construct it attempts to represent (<italic>face</italic> validity). <italic>Construct</italic> validity reflects the ability of the instrument to actually measure what it intends to measure (ie, the difference in performance between groups of users with different levels of experience in reality). <italic>Concurrent</italic> validity reflects the correlation between performance on the serious game and their performance on an instrument believed to measure the same construct (eg, a simulator or course). The ultimate goal is to prove a game&#8217;s <italic>predictive</italic> validity: does performance in the game lead to better outcomes in reality? Most validation research currently published in the medical domain uses these concepts [<xref ref-type="bibr" rid="ref29">29</xref>]. For individual cases, relevance of specific validity types may differ. When considering mHealth apps in general, content validity may be the sole source of validity.</p>
                <p>Validity research is frequently a long and costly enterprise. Many newly developed serious games have therefore not yet undergone validity research [<xref ref-type="bibr" rid="ref8">8</xref>]. The framework therefore determines a number of steps to pre-assess a serious game&#8217;s potential as a valid instrument, with regard to its design and initial testing phases. This encompasses the involvement of user groups, content experts, or educationalists in the design (if relevant to the game&#8217;s purpose). Next, if a game has undergone user testing and stability testing, the game is more likely to have higher face- and content validity.</p>
            </sec>
            <sec>
                <title>Data Protection</title>
                <p>Threats to user privacy are imminent in electronic and mobile health apps, especially when patient-specific data are measured or entered in the game [<xref ref-type="bibr" rid="ref15">15</xref>]. This considers data &#8220;at rest&#8221; on devices or servers, as well as data &#8220;in transit&#8221;. It must be clear whether data is collected by the game, who owns the data and whether users can request to remove their data. Storage and analysis of personal data should be disclosed to users and must be in conformity with the laws applied in countries the serious game is distributed in. Special care must be taken if patient information is collected. These items are in general conformity with the requirements for mHealth apps described in Albrecht&#8217;s framework [<xref ref-type="bibr" rid="ref18">18</xref>].</p>
            </sec>
        </sec>
        <sec sec-type="discussion">
            <title>Discussion</title>
            <p>When using serious games in health care, end users (clinicians, patients, or educators) must decide whether games are safe and effective enough to be used for their intended purposes. In order to do so, they need consistent, transparent, and reliable assessments. Are applied games really stating their claim in this field? In the framework described in this article, both developers and end users are supported in assessing relevance, validity, and data safety of an applied game. In order to become a &#8220;qualified game&#8221;, developers should disclose comprehensive information on their products and claims. They must provide transparency to meet the standards. The <italic>Journal of Medical Internet Research</italic> and the Dutch Society for Simulation in Healthcare [<xref ref-type="bibr" rid="ref16">16</xref>] have launched an international peer-reviewing initiative for serious games in health care.</p>
            <p>The safe application of technology-enhanced solution remains the responsibility of the health care provider. Choosing if a serious game answers to the user&#8217;s needs, can be based on information concerning 5 main areas described in this article. The majority of the items cannot be assessed using objective parameters. For instance, claiming a specific serious game&#8217;s predictive validity should be supported by solid evidence. A comprehensive evaluation by a panel of experts in the form of a quality label could form a more practical solution.</p>
            <p>Guidelines have been recently published reporting standards to support clinicians and patients in distinguishing high quality mhealth apps [<xref ref-type="bibr" rid="ref17">17</xref>-<xref ref-type="bibr" rid="ref19">19</xref>] and medical websites [<xref ref-type="bibr" rid="ref30">30</xref>]. These standards form the basis for the framework described in this article. These standards have two important shortcomings when it comes to games. First, explicit information on a serious game&#8217;s content and didactic features is required, as the external purpose of a serious game is frequently less obvious to the user than in the case of mHealth apps. Second, serious games require additional validation steps (eg, construct and predictive validity), compared to non-interactive information platforms. Gameplay is dynamic and learning goals in gameplay are often not disclosed to the user. In fact, the user learns by playing the game, whereas discovery in itself may be part of the gameplay. Disclosing learning goals would thus be counterproductive.</p>
            <p>There are several limitations to the framework described in this study. It considers validity of the serious game&#8217;s content and its didactic functionality. Validity does not predict a game&#8217;s success nor its attractiveness to the user, which also depend on its entertainment capability and distribution method [<xref ref-type="bibr" rid="ref31">31</xref>]. It does not wish to objectify which game is most fun, but merely which game is most valid. A second consideration is that in the scientific field of validity research in medicine, validity concepts other than the one used in this framework have been proposed [<xref ref-type="bibr" rid="ref32">32</xref>]. The &#8220;classical&#8221; validity concepts (content-, face-, construct-, concurrent-, and predictive validity) have been most frequently used in validity research in medicine and therefore the most logical to encompass in the framework presented in this article [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>].</p>
            <p>In summary, this consensus-based tool provides the end users the support required when assessing the effectiveness and relevance of serious games in health care. An FDA-approval or CE-mark is simply insufficient for this purpose. In order to prevent wrongful application and data theft of unsuspecting patients or medical students, this information on medical serious games should become publically available to all end users. This will aid the prescription of safe and effective games to patients and the implementation of games into educational programs.</p>
        </sec>
    </body>
    <back>
        <ack>
            <p>The authors received funding from the Patient safety project (grant ref PID 101060), of the <italic>Pieken in de Delta</italic>-program by the Dutch Ministry of Economic Affairs, Agriculture and Innovation, the city of Utrecht and the province of Utrecht, the Netherlands. The funding agency had no role in the preparation of the consensus tool, nor the preparation of the manuscript.</p>
            <p>The Committee for Serious Gaming committee of the Dutch Society for Simulation in Healthcare (DSSH) consist of the following members: M Graafland (Department of Surgery, Academic Medical Centre, Amsterdam, The Netherlands); MEW Dankbaar (Department of Medical Education (Desiderius School), Erasmus University Medical Centre, Rotterdam, The Netherlands); A Mert (National Military Rehabilitation Centre, Doorn, The Netherlands); J Lagro (Department of Geriatrics, Radboud University Medical Centre, Nijmegen, The Netherlands); LD de Wit-Zuurendonk (Department of Obstetrics and Gynaecology, M&#225;xima Medical Centre, Veldhoven, The Netherlands); SCE Schuit (Departments of Emergency Medicine and Internal Medicine, Erasmus University Medical Centre, Rotterdam, The Netherlands); A Schaafstal (Department of ICT, Windesheim University of Applied Sciences, Zwolle, The Netherlands); and MP Schijven (Department of Surgery, Academic Medical Centre, Amsterdam, The Netherlands).</p>
        </ack>
        <fn-group>
            <fn fn-type="conflict">
                <p>None declared.</p>
            </fn>
        </fn-group>
        <ref-list>
            <ref id="ref1">
                <label>1</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kueider</surname>
                            <given-names>AM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Parisi</surname>
                            <given-names>JM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Gross</surname>
                            <given-names>AL</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Rebok</surname>
                            <given-names>GW</given-names>
                        </name>
                    </person-group>
                    <article-title>Computerized cognitive training with older adults: a systematic review</article-title>
                    <source>PLoS One</source>
                    <year>2012</year>
                    <volume>7</volume>
                    <issue>7</issue>
                    <fpage>e40588</fpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://dx.plos.org/10.1371/journal.pone.0040588" />
                    </comment>
                    <pub-id pub-id-type="doi">10.1371/journal.pone.0040588</pub-id>
                    <pub-id pub-id-type="medline">22792378</pub-id>
                    <pub-id pub-id-type="pii">PONE-D-12-10422</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3394709</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref2">
                <label>2</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Gamito</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Oliveira</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lopes</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brito</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Morais</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Silva</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Silva</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Rebelo</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bastos</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Deus</surname>
                            <given-names>A</given-names>
                        </name>
                    </person-group>
                    <article-title>Executive functioning in alcoholics following an mHealth cognitive stimulation program: randomized controlled trial</article-title>
                    <source>J Med Internet Res</source>
                    <year>2014</year>
                    <volume>16</volume>
                    <issue>4</issue>
                    <fpage>e102</fpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.jmir.org/2014/4/e102/" />
                    </comment>
                    <pub-id pub-id-type="doi">10.2196/jmir.2923</pub-id>
                    <pub-id pub-id-type="medline">24742381</pub-id>
                    <pub-id pub-id-type="pii">v16i4e102</pub-id>
                    <pub-id pub-id-type="pmcid">PMC4019779</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref3">
                <label>3</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Prange</surname>
                            <given-names>GB</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kottink</surname>
                            <given-names>AI</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Buurke</surname>
                            <given-names>JH</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Eckhardt</surname>
                            <given-names>MM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>van Keulen-Rouweler</surname>
                            <given-names>BJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ribbers</surname>
                            <given-names>GM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Rietman</surname>
                            <given-names>JS</given-names>
                        </name>
                    </person-group>
                    <article-title>The Effect of Arm Support Combined With Rehabilitation Games on Upper-Extremity Function in Subacute Stroke: A Randomized Controlled Trial</article-title>
                    <source>Neurorehabil Neural Repair</source>
                    <year>2014</year>
                    <month>05</month>
                    <day>29</day>
                    <pub-id pub-id-type="doi">10.1177/1545968314535985</pub-id>
                    <pub-id pub-id-type="medline">24878589</pub-id>
                    <pub-id pub-id-type="pii">1545968314535985</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref4">
                <label>4</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Majumdar</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Koch</surname>
                            <given-names>PA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lee</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Contento</surname>
                            <given-names>IR</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Islas-Ramos</surname>
                            <given-names>AD</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Fu</surname>
                            <given-names>D</given-names>
                        </name>
                    </person-group>
                    <article-title>&#34;Creature-101&#34;: A Serious Game to Promote Energy Balance-Related Behaviors Among Middle School Adolescents</article-title>
                    <source>Games Health J</source>
                    <year>2013</year>
                    <month>10</month>
                    <volume>2</volume>
                    <issue>5</issue>
                    <fpage>280</fpage>
                    <lpage>290</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24761326" />
                    </comment>
                    <pub-id pub-id-type="doi">10.1089/g4h.2013.0045</pub-id>
                    <pub-id pub-id-type="medline">24761326</pub-id>
                    <pub-id pub-id-type="pii">10.1089/g4h.2013.0045</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3833376</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref5">
                <label>5</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cooper</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cooper</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Milton</surname>
                            <given-names>B</given-names>
                        </name>
                    </person-group>
                    <article-title>Technology-based approaches to patient education for young people living with diabetes: a systematic literature review</article-title>
                    <source>Pediatr Diabetes</source>
                    <year>2009</year>
                    <month>11</month>
                    <volume>10</volume>
                    <issue>7</issue>
                    <fpage>474</fpage>
                    <lpage>83</lpage>
                    <pub-id pub-id-type="doi">10.1111/j.1399-5448.2009.00509.x</pub-id>
                    <pub-id pub-id-type="medline">19490492</pub-id>
                    <pub-id pub-id-type="pii">PDI509</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref6">
                <label>6</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Read</surname>
                            <given-names>JL</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Shortell</surname>
                            <given-names>SM</given-names>
                        </name>
                    </person-group>
                    <article-title>Interactive games to promote behavior change in prevention and treatment</article-title>
                    <source>JAMA</source>
                    <year>2011</year>
                    <month>04</month>
                    <day>27</day>
                    <volume>305</volume>
                    <issue>16</issue>
                    <fpage>1704</fpage>
                    <lpage>5</lpage>
                    <pub-id pub-id-type="doi">10.1001/jama.2011.408</pub-id>
                    <pub-id pub-id-type="medline">21447802</pub-id>
                    <pub-id pub-id-type="pii">jama.2011.408</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref7">
                <label>7</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kato</surname>
                            <given-names>PM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cole</surname>
                            <given-names>SW</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bradlyn</surname>
                            <given-names>AS</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pollock</surname>
                            <given-names>BH</given-names>
                        </name>
                    </person-group>
                    <article-title>A video game improves behavioral outcomes in adolescents and young adults with cancer: a randomized trial</article-title>
                    <source>Pediatrics</source>
                    <year>2008</year>
                    <month>08</month>
                    <volume>122</volume>
                    <issue>2</issue>
                    <fpage>e305</fpage>
                    <lpage>17</lpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://pediatrics.aappublications.org/cgi/pmidlookup?view=long&#38;pmid=18676516" />
                    </comment>
                    <pub-id pub-id-type="doi">10.1542/peds.2007-3134</pub-id>
                    <pub-id pub-id-type="medline">18676516</pub-id>
                    <pub-id pub-id-type="pii">122/2/e305</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref8">
                <label>8</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Graafland</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Schraagen</surname>
                            <given-names>JM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Schijven</surname>
                            <given-names>MP</given-names>
                        </name>
                    </person-group>
                    <article-title>Systematic review of serious games for medical education and surgical skills training</article-title>
                    <source>Br J Surg</source>
                    <year>2012</year>
                    <month>10</month>
                    <volume>99</volume>
                    <issue>10</issue>
                    <fpage>1322</fpage>
                    <lpage>30</lpage>
                    <pub-id pub-id-type="doi">10.1002/bjs.8819</pub-id>
                    <pub-id pub-id-type="medline">22961509</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref9">
                <label>9</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Buttussi</surname>
                            <given-names>F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pellis</surname>
                            <given-names>T</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Cabas Vidani</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pausler</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Carchietti</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Chittaro</surname>
                            <given-names>L</given-names>
                        </name>
                    </person-group>
                    <article-title>Evaluation of a 3D serious game for advanced life support retraining</article-title>
                    <source>Int J Med Inform</source>
                    <year>2013</year>
                    <month>09</month>
                    <volume>82</volume>
                    <issue>9</issue>
                    <fpage>798</fpage>
                    <lpage>809</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.ijmedinf.2013.05.007</pub-id>
                    <pub-id pub-id-type="medline">23763908</pub-id>
                    <pub-id pub-id-type="pii">S1386-5056(13)00114-7</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref10">
                <label>10</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Lagro</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>van de Pol</surname>
                            <given-names>MH</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Laan</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Huijbregts-Verheyden</surname>
                            <given-names>FJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Fluit</surname>
                            <given-names>LC</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Olde Rikkert</surname>
                            <given-names>MG</given-names>
                        </name>
                    </person-group>
                    <article-title>A Randomized Controlled Trial on Teaching Geriatric Medical Decision Making and Cost Consciousness With the Serious Game GeriatriX</article-title>
                    <source>J Am Med Dir Assoc</source>
                    <year>2014</year>
                    <month>06</month>
                    <day>6</day>
                    <pub-id pub-id-type="doi">10.1016/j.jamda.2014.04.011</pub-id>
                    <pub-id pub-id-type="medline">24913210</pub-id>
                    <pub-id pub-id-type="pii">S1525-8610(14)00229-1</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref11">
                <label>11</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Forsberg</surname>
                            <given-names>A</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Nilsag&#229;rd</surname>
                            <given-names>Y</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bostr&#246;m</surname>
                            <given-names>K</given-names>
                        </name>
                    </person-group>
                    <article-title>Perceptions of using videogames in rehabilitation: a dual perspective of people with multiple sclerosis and physiotherapists</article-title>
                    <source>Disabil Rehabil</source>
                    <year>2014</year>
                    <month>05</month>
                    <day>16</day>
                    <fpage>1</fpage>
                    <lpage>7</lpage>
                    <pub-id pub-id-type="doi">10.3109/09638288.2014.918196</pub-id>
                    <pub-id pub-id-type="medline">24833535</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref12">
                <label>12</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Graafland</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Vollebergh</surname>
                            <given-names>MF</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lagarde</surname>
                            <given-names>SM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>van Haperen</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bemelman</surname>
                            <given-names>WA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Schijven</surname>
                            <given-names>MP</given-names>
                        </name>
                    </person-group>
                    <article-title>A Serious Game Can Be a Valid Method to Train Clinical Decision-Making in Surgery</article-title>
                    <source>World J Surg</source>
                    <year>2014</year>
                    <month>08</month>
                    <day>27</day>
                    <pub-id pub-id-type="doi">10.1007/s00268-014-2743-4</pub-id>
                    <pub-id pub-id-type="medline">25159118</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref13">
                <label>13</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>van Velsen</surname>
                            <given-names>L</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Beaujean</surname>
                            <given-names>DJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>van Gemert-Pijnen</surname>
                            <given-names>JE</given-names>
                        </name>
                    </person-group>
                    <article-title>Why mobile health app overload drives us crazy, and how to restore the sanity</article-title>
                    <source>BMC Med Inform Decis Mak</source>
                    <year>2013</year>
                    <volume>13</volume>
                    <fpage>23</fpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.biomedcentral.com/1472-6947/13/23" />
                    </comment>
                    <pub-id pub-id-type="doi">10.1186/1472-6947-13-23</pub-id>
                    <pub-id pub-id-type="medline">23399513</pub-id>
                    <pub-id pub-id-type="pii">1472-6947-13-23</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3621678</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref14">
                <label>14</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Connolly</surname>
                            <given-names>TM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Boyle</surname>
                            <given-names>EA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>MacArthur</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hainey</surname>
                            <given-names>T</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Boyle</surname>
                            <given-names>JM</given-names>
                        </name>
                    </person-group>
                    <article-title>A systematic literature review of empirical evidence on computer games and serious games</article-title>
                    <source>Computers &#38; Education</source>
                    <year>2012</year>
                    <month>9</month>
                    <volume>59</volume>
                    <issue>2</issue>
                    <fpage>661</fpage>
                    <lpage>686</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.compedu.2012.03.004</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref15">
                <label>15</label>
                <nlm-citation citation-type="confproc">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kotz</surname>
                            <given-names>D</given-names>
                        </name>
                    </person-group>
                    <article-title>A threat taxonomy for mHealth privacy</article-title>
                    <source>Crowcroft J, Manjunath D, Misra A.  editors Third International Conference on Communication Systems and Networks (COMSNETS 2011)</source>
                    <year>2011</year>
                    <conf-name>Third International Conference on Communication Systems and Networks</conf-name>
                    <conf-date>2011</conf-date>
                    <conf-loc>New York, NY</conf-loc>
                    <publisher-name>IEEE</publisher-name>
                    <fpage>1</fpage>
                    <pub-id pub-id-type="doi">10.1109/COMSNETS.2011.5716518</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref16">
                <label>16</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <collab>Dutch Society for Simulation in Healthcare</collab>
                    </person-group>
                    <article-title>Official Website Internet</article-title>
                    <source>DSSH official website</source>
                    <year>2014</year>
                    <fpage>2014</fpage>
                </nlm-citation>
            </ref>
            <ref id="ref17">
                <label>17</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Lewis</surname>
                            <given-names>TL</given-names>
                        </name>
                    </person-group>
                    <article-title>A systematic self-certification model for mobile medical apps</article-title>
                    <source>J Med Internet Res</source>
                    <year>2013</year>
                    <volume>15</volume>
                    <issue>4</issue>
                    <fpage>e89</fpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.jmir.org/2013/4/e89/" />
                    </comment>
                    <pub-id pub-id-type="doi">10.2196/jmir.2446</pub-id>
                    <pub-id pub-id-type="medline">23615332</pub-id>
                    <pub-id pub-id-type="pii">v15i4e89</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3636320</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref18">
                <label>18</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Albrecht</surname>
                            <given-names>UV</given-names>
                        </name>
                    </person-group>
                    <article-title>Transparency of health-apps for trust and decision making</article-title>
                    <source>J Med Internet Res</source>
                    <year>2013</year>
                    <volume>15</volume>
                    <issue>12</issue>
                    <fpage>e277</fpage>
                    <comment>
                        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.jmir.org/2013/12/e277/" />
                    </comment>
                    <pub-id pub-id-type="doi">10.2196/jmir.2981</pub-id>
                    <pub-id pub-id-type="medline">24449711</pub-id>
                    <pub-id pub-id-type="pii">v15i12e277</pub-id>
                    <pub-id pub-id-type="pmcid">PMC3958693</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref19">
                <label>19</label>
                <nlm-citation citation-type="web">
                    <source>Journal of Medical Internet Research (JMIR)</source>
                    <year>2014</year>
                    <access-date>2014-11-04</access-date>
                    <comment>Apps Peer-Review launched Internet<ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://mhealth.jmir.org/announcement/view/67">http://mhealth.jmir.org/announcement/view/67</ext-link>
                    </comment>
                    <pub-id pub-id-type="other">6TpSDiKIJ</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref20">
                <label>20</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Michael</surname>
                            <given-names>DR</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Chen</surname>
                            <given-names>S</given-names>
                        </name>
                    </person-group>
                    <source>Serious Games: Games That Educate, Train, and Inform</source>
                    <year>2006</year>
                    <publisher-loc>Boston, MA</publisher-loc>
                    <publisher-name>Course Technology PTR</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref21">
                <label>21</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Jalink</surname>
                            <given-names>MB</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Goris</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Heineman</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Pierie</surname>
                            <given-names>JP</given-names>
                        </name>
                        <name name-style="western">
                            <surname>ten Cate Hoedemaker</surname>
                            <given-names>HO</given-names>
                        </name>
                    </person-group>
                    <article-title>Construct and concurrent validity of a Nintendo Wii video game made for training basic laparoscopic skills</article-title>
                    <source>Surg Endosc</source>
                    <year>2014</year>
                    <month>02</month>
                    <volume>28</volume>
                    <issue>2</issue>
                    <fpage>537</fpage>
                    <lpage>42</lpage>
                    <pub-id pub-id-type="doi">10.1007/s00464-013-3199-6</pub-id>
                    <pub-id pub-id-type="medline">24061627</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref22">
                <label>22</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Badurdeen</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Abdul-Samad</surname>
                            <given-names>O</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Story</surname>
                            <given-names>G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Wilson</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Down</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Harris</surname>
                            <given-names>A</given-names>
                        </name>
                    </person-group>
                    <article-title>Nintendo Wii video-gaming ability predicts laparoscopic skill</article-title>
                    <source>Surg Endosc</source>
                    <year>2010</year>
                    <month>08</month>
                    <volume>24</volume>
                    <issue>8</issue>
                    <fpage>1824</fpage>
                    <lpage>8</lpage>
                    <pub-id pub-id-type="doi">10.1007/s00464-009-0862-z</pub-id>
                    <pub-id pub-id-type="medline">20108147</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref23">
                <label>23</label>
                <nlm-citation citation-type="web">
                    <source>SCORM</source>
                    <access-date>2014-11-04</access-date>
                    <comment>SCORM Explained Internet<ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://scorm.com/scorm-explained/">http://scorm.com/scorm-explained/</ext-link>
                    </comment>
                    <pub-id pub-id-type="other">6TpSOWvl4</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref24">
                <label>24</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Anderson</surname>
                            <given-names>LW</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Krathwohl</surname>
                            <given-names>DR</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bloom</surname>
                            <given-names>Bs</given-names>
                        </name>
                    </person-group>
                    <source>A taxonomy for learning, teaching, and assessing: a revision of Bloom&#039;s taxonomy of educational objectives</source>
                    <year>2001</year>
                    <publisher-loc>New York, NY</publisher-loc>
                    <publisher-name>Longman</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref25">
                <label>25</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Van Staalduinen</surname>
                            <given-names>JP</given-names>
                        </name>
                        <name name-style="western">
                            <surname>de Freitas</surname>
                            <given-names>S</given-names>
                        </name>
                    </person-group>
                    <person-group person-group-type="editor">
                        <name name-style="western">
                            <surname>Khine</surname>
                            <given-names>MS</given-names>
                        </name>
                    </person-group>
                    <article-title>A Game-Based Learning Framework: Linking Game DesignLearning Outcomes</article-title>
                    <source>Learning to Play</source>
                    <year>2011</year>
                    <publisher-loc>New York, NY</publisher-loc>
                    <publisher-name>Peter Lang Publishing, Inc</publisher-name>
                    <fpage>29</fpage>
                </nlm-citation>
            </ref>
            <ref id="ref26">
                <label>26</label>
                <nlm-citation citation-type="book">
                    <person-group person-group-type="author">
                        <collab>American Educational Research Association, American Psychological Association, National Council on Measurement in Education</collab>
                    </person-group>
                    <source>Standards for educational and psychological testing</source>
                    <year>1999</year>
                    <publisher-loc>Washington, DC</publisher-loc>
                    <publisher-name>American Educational Research Association</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref27">
                <label>27</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Schijven</surname>
                            <given-names>MP</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jakimowicz</surname>
                            <given-names>JJ</given-names>
                        </name>
                    </person-group>
                    <article-title>Validation of virtual reality simulators: Key to the successful integration of a novel teaching technology into minimal access surgery</article-title>
                    <source>Minim Invasive Ther Allied Technol</source>
                    <year>2005</year>
                    <volume>14</volume>
                    <issue>4</issue>
                    <fpage>244</fpage>
                    <lpage>6</lpage>
                    <pub-id pub-id-type="doi">10.1080/13645700500221881</pub-id>
                    <pub-id pub-id-type="medline">16754170</pub-id>
                    <pub-id pub-id-type="pii">T1205666582U2174</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref28">
                <label>28</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Gallagher</surname>
                            <given-names>AG</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ritter</surname>
                            <given-names>EM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Satava</surname>
                            <given-names>RM</given-names>
                        </name>
                    </person-group>
                    <article-title>Fundamental principles of validation, and reliability: rigorous science for the assessment of surgical education and training</article-title>
                    <source>Surg Endosc</source>
                    <year>2003</year>
                    <month>10</month>
                    <volume>17</volume>
                    <issue>10</issue>
                    <fpage>1525</fpage>
                    <lpage>9</lpage>
                    <pub-id pub-id-type="doi">10.1007/s00464-003-0035-4</pub-id>
                    <pub-id pub-id-type="medline">14502403</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref29">
                <label>29</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cook</surname>
                            <given-names>DA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Zendejas</surname>
                            <given-names>B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hamstra</surname>
                            <given-names>SJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hatala</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brydges</surname>
                            <given-names>R</given-names>
                        </name>
                    </person-group>
                    <article-title>What counts as validity evidence? Examples and prevalence in a systematic review of simulation-based assessment</article-title>
                    <source>Adv Health Sci Educ Theory Pract</source>
                    <year>2014</year>
                    <month>05</month>
                    <volume>19</volume>
                    <issue>2</issue>
                    <fpage>233</fpage>
                    <lpage>50</lpage>
                    <pub-id pub-id-type="doi">10.1007/s10459-013-9458-4</pub-id>
                    <pub-id pub-id-type="medline">23636643</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref30">
                <label>30</label>
                <nlm-citation citation-type="web">
                    <source>Health on the Net Foundation Code of Conduct (HONcode)</source>
                    <access-date>2014-11-04</access-date>
                    <comment>Operational definition of the HONcode principles Internet<ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.hon.ch/HONcode/Webmasters/Guidelines/guidelines.html">http://www.hon.ch/HONcode/Webmasters/Guidelines/guidelines.html</ext-link>
                    </comment>
                    <pub-id pub-id-type="other">6TpScFyun</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref31">
                <label>31</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Bellotti</surname>
                            <given-names>F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kapralos</surname>
                            <given-names>B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lee</surname>
                            <given-names>K</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Moreno-Ger</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Berta</surname>
                            <given-names>R</given-names>
                        </name>
                    </person-group>
                    <article-title>Assessment in and of Serious Games: An Overview</article-title>
                    <source>Advances in Human-Computer Interaction</source>
                    <year>2013</year>
                    <volume>2013</volume>
                    <fpage>1</fpage>
                    <lpage>11</lpage>
                    <pub-id pub-id-type="doi">10.1155/2013/136864</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref32">
                <label>32</label>
                <nlm-citation citation-type="journal">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cook</surname>
                            <given-names>DA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Beckman</surname>
                            <given-names>TJ</given-names>
                        </name>
                    </person-group>
                    <article-title>Current concepts in validity and reliability for psychometric instruments: theory and application</article-title>
                    <source>Am J Med</source>
                    <year>2006</year>
                    <month>02</month>
                    <volume>119</volume>
                    <issue>2</issue>
                    <fpage>166.e7</fpage>
                    <lpage>16</lpage>
                    <pub-id pub-id-type="doi">10.1016/j.amjmed.2005.10.036</pub-id>
                    <pub-id pub-id-type="medline">16443422</pub-id>
                    <pub-id pub-id-type="pii">S0002-9343(05)01037-5</pub-id>
                </nlm-citation>
            </ref>
        </ref-list>
    </back>
</article>
