CINXE.COM

<!DOCTYPE html><html lang="en"><head><title data-react-helmet="true">How does ask a question work?</title> <meta data-react-helmet="true" name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5, viewport-fit=cover"/><meta data-react-helmet="true" property="og:site_name" content="scite.ai"/><meta data-react-helmet="true" property="og:type" content="website"/><meta data-react-helmet="true" property="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:site" content="@scite"/><meta data-react-helmet="true" property="title" content="How does ask a question work?"/><meta data-react-helmet="true" property="og:title" content="How does ask a question work?"/><meta data-react-helmet="true" property="twitter:title" content="How does ask a question work?"/><meta data-react-helmet="true" property="og:image" content="http://image.thum.io/get/ogImage/https://scite.ai/blog/2022-11-16_how_does_ask_a_question_work?bannerClosed=true&v=3"/><meta data-react-helmet="true" property="twitter:image" content="http://image.thum.io/get/ogImage/https://scite.ai/blog/2022-11-16_how_does_ask_a_question_work?bannerClosed=true&v=3"/><meta data-react-helmet="true" name="description" content="Ask a question helps you ask simple, natural questions and get answers extracted from more than 32 million full texts (closed and open access). How does it work?"/><meta data-react-helmet="true" property="og:description" content="Ask a question helps you ask simple, natural questions and get answers extracted from more than 32 million full texts (closed and open access). How does it work?"/><meta data-react-helmet="true" property="twitter:description" content="Ask a question helps you ask simple, natural questions and get answers extracted from more than 32 million full texts (closed and open access). How does it work?"/><meta data-react-helmet="true" property="og:url" content="https://scite.ai/blog/2022-11-16_how_does_ask_a_question_work"/> <link data-react-helmet="true" rel="preconnect" href="https://cdn.scite.ai" crossorigin="true"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.googleapis.com"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.gstatic.com" crossorigin="true"/><link data-react-helmet="true" rel="shortcut icon" href="https://cdn.scite.ai/assets/images/favicon.ico"/><link data-react-helmet="true" rel="stylesheet preload prefetch" as="style" href="https://cdn.scite.ai/assets/css/line-awesome-font-awesome.min.css?v=2"/><link data-react-helmet="true" rel="preload" as="style" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@200;300;400;500;600&display=swap"/><link data-react-helmet="true" rel="stylesheet" as="style" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@200;300;400;500;600&display=swap" media="print" onload="this.media='all'"/><link data-react-helmet="true" rel="canonical" href="https://scite.ai/blog/2022-11-16_how_does_ask_a_question_work"/> <script data-react-helmet="true" src="https://www.recaptcha.net/recaptcha/api.js?render=6LcnMGsqAAAAAJLHdGuust5QwzLk-asKPW5MWbAx"></script> <link data-chunk="main" rel="stylesheet" href="https://cdn.scite.ai/assets/dist/main.2a4a881c9257077826c8.css"> <link data-chunk="BlogApp" rel="stylesheet" href="https://cdn.scite.ai/assets/dist/3115.b46e5cef2610b838fe16.css"> </head> <body><div id="scite-app"><div class="BlogApp__pageLayout___WNcxh"><div><nav class="NavBar__navBarContainer___skQfv"><div class="NavBar__navBar___ELwVq NavBar__notStatic___bzily" style="transform:translate3d(0,0%,0)"><div class="NavBar__navContents___X2u7r"><a class="NavBar__logo___aYKoi" href="/"><img alt="scite logo" width="115px" height="38px" class="NavBar__logoDesktop___Cwo_2" style="filter:contrast(2.15)" src="https://cdn.scite.ai/assets/images/logo-blue.svg"/><img alt="scite logo" width="64px" height="38px" class="NavBar__logoMobile___dA77Y" style="filter:contrast(2.15)" src="https://cdn.scite.ai/assets/images/logo-blue.svg"/></a><form class="SearchInput__searchInput___ze14_ NavBar__searchBox___l1QGV NavBar__searchBarBorder___xB_jN SearchBox__input___Q1BdB"><div></div><input aria-label="search" data-showdropdown="true" autoComplete="off" class="SearchInput__input___D08BX" type="text" placeholder="Search by title, author, keywords or DOI" value=""/><div class="SearchInput__button___L2ljx NavBar__searchButton___ZfbtP" aria-label="search-button" role="button" tabindex="0"><i class="icon icon-search SearchInput__icon___sEZb9"></i></div></form><i class="NavBar__mobileSearchIcon___Ftzic icon icon-search" role="button" tabindex="0" aria-label="Search"></i><div class="NavBar__rightContent___bJvfl"><div class="NavBar__navButtonsLayout___SnYwi"><div class="NavButtons__navButtons___uY5SG"><div class="NavButtons__mobileHide___bSlk6"><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/assistant">Assistant</a></div><div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button">Product<div class="OpenerIcon__folder___jiKx6 false HoverLink__icon___H8E9g"><i class="OpenerIcon__angleDown___QHC0i icon icon-down-carret"></i></div></a></div></div><div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button">Solutions<div class="OpenerIcon__folder___jiKx6 false HoverLink__icon___H8E9g"><i class="OpenerIcon__angleDown___QHC0i icon icon-down-carret"></i></div></a></div></div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/pricing">Pricing</a></div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/blog">Blog</a></div></div></div></div></div></div></div></nav><div><nav class="NavBar__navBarContainer___skQfv NavBar__spacer___BOOCE" style="visibility:hidden"><div class="NavBar__navBar___ELwVq NavBar__notStatic___bzily NavBar__spacer___BOOCE" style="transform:translate3d(0,0%,0)"><div class="NavBar__navContents___X2u7r"><a class="NavBar__logo___aYKoi" href="/"><img alt="scite logo" width="115px" height="38px" class="NavBar__logoDesktop___Cwo_2" style="filter:contrast(2.15)" src="https://cdn.scite.ai/assets/images/logo-blue.svg"/><img alt="scite logo" width="64px" height="38px" class="NavBar__logoMobile___dA77Y" style="filter:contrast(2.15)" src="https://cdn.scite.ai/assets/images/logo-blue.svg"/></a><form class="SearchInput__searchInput___ze14_ NavBar__searchBox___l1QGV NavBar__searchBarBorder___xB_jN SearchBox__input___Q1BdB"><div></div><input aria-label="search" data-showdropdown="true" autoComplete="off" class="SearchInput__input___D08BX" type="text" placeholder="Search by title, author, keywords or DOI" value=""/><div class="SearchInput__button___L2ljx NavBar__searchButton___ZfbtP" aria-label="search-button" role="button" tabindex="0"><i class="icon icon-search SearchInput__icon___sEZb9"></i></div></form><i class="NavBar__mobileSearchIcon___Ftzic icon icon-search" role="button" tabindex="0" aria-label="Search"></i><div class="NavBar__rightContent___bJvfl"><div class="NavBar__navButtonsLayout___SnYwi"><div class="NavButtons__navButtons___uY5SG"><div class="NavButtons__mobileHide___bSlk6"><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/assistant">Assistant</a></div><div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button">Product<div class="OpenerIcon__folder___jiKx6 false HoverLink__icon___H8E9g"><i class="OpenerIcon__angleDown___QHC0i icon icon-down-carret"></i></div></a></div></div><div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button">Solutions<div class="OpenerIcon__folder___jiKx6 false HoverLink__icon___H8E9g"><i class="OpenerIcon__angleDown___QHC0i icon icon-down-carret"></i></div></a></div></div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/pricing">Pricing</a></div><div class="HoverLink__container___PEeZm"><a class="HoverLink__link___wZgvn Link__link___plGt2 HoverLink__link___wZgvn" tabindex="0" role="button" href="/blog">Blog</a></div></div></div></div></div></div></div></nav></div></div><div class="BlogApp__layout___GW_mz"><div class="BlogApp__toc___m8r1Q"><nav class="TableOfContents__layout___xkXBj"><a class="TableOfContents__category___kifGa" tabindex="0">case study</a><a class="TableOfContents__category___kifGa" tabindex="0">development</a><a class="TableOfContents__category___kifGa" tabindex="0">new feature</a><a class="TableOfContents__category___kifGa" tabindex="0">news</a><a class="TableOfContents__category___kifGa" tabindex="0">other</a><a class="TableOfContents__category___kifGa TableOfContents__selectedCategory___SLyss" tabindex="0">research</a><a class="TableOfContents__category___kifGa" tabindex="0">tutorial</a></nav></div><div class="BlogApp__post___WSrvx"><span class="BlogApp__blogLabel___FiTzi"><a class="Link__link___plGt2 Link__underline___K1Y7j" tabindex="0" href="/blog">Blog</a> / <a class="Link__link___plGt2 Link__underline___K1Y7j" tabindex="0" href="/blog?category=Research">Research</a></span><h4 class="Header__h4___DEzov BlogApp__mainHeader___gEkdP">How does ask a question work?</h4><h6 class="Header__h6___QzBvp BlogApp__subheader___LFEiF">Ask a question is an extractive question answering system on over 1.2bn citation statements.</h6><span class="BlogApp__date___XEUtZ">Wed Nov 16 2022</span><div class="BlogApp__postInfo___dw_bY"><span>Published by scite</span><span class="BlogApp__categories___RuqWT"><span class="Categories__category___BsVbr" tabindex="0" role="button">Research</span></span><div class="ShareButtons__shareButtons___TodEt BlogApp__shareButtons___Qk06V"><span>Share</span><span class="ShareButtons__icons___mhgN8"><a class="ShareButtons__icon___Ve2y5" href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__twitter___NsyXj icon icon-twitter-new"></i></a><a class="ShareButtons__icon___Ve2y5" href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__linkedin___nS70z icon icon-linkedin-new"></i></a><a class="ShareButtons__icon___Ve2y5" href="https://facebook.com/share.php?u=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__facebook___X8lcc icon icon-facebook-new"></i></a></span></div></div><div><p class="Paragraph__body____m5FY">Recently we released an early preview of a question answering (QA) system built on top of our data (see <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://scite.ai/data-and-services" tabindex="0" role="link">https://scite.ai/data-and-services</a> for a more comprehensive picture of what our data is). This short article will explain how it <em>currently</em> works as of November 16th 2022 and a little bit about our current plans for expanding it</p><p class="Paragraph__body____m5FY">This is a fairly technical post so feel free to reach out to us at customersupport@researchsolutions.com if you have questions.</p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="why-build-a-scientific-question-answering-system">Why build a scientific question answering system?</h4><p class="Paragraph__body____m5FY">While we have a pretty powerful search system that already allows you to search 1.2bn citation statements extracted from 32 million full texts (both closed and open access!)... search isn't always the easiest way to get after the kinds of information you are looking for. This is especially true since we use a "lexical" (read word matching)-based search system using elasticsearch - so if you don't formulate your search query exactly with the words you want matched you might not get what you are looking for. In addition, much of the "information seeking" experiences we have when looking at science comes in the form of questions we need answers to. Therefore, we thought it might help our users discover science-backed answers if we build a QA system over all our data!</p><p class="Paragraph__body____m5FY"><strong>We are building a large-scale scientific question answering system to help users get science-based answers to questions</strong>.</p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="what-is-question-answering">What is question answering?</h4><p class="Paragraph__body____m5FY">Question answering is a popular task in the natural language processing community that seeks to provide an answer for a given question posed by the user. There are plenty of formulations of question answering from multiple choice, closed book, and extractive qa to open domain and multi-hop qa. It is a very popular task; for a great survey on the recent explosion of datasets alone see <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://arxiv.org/abs/2107.12708" tabindex="0" role="link">QA Dataset Explosion: A Taxonomy of NLP Resources for Question Answering and Reading Comprehension</a>.</p><p class="Paragraph__body____m5FY">Our approach is <em>open-domain extractive question answering.</em> This means that we are looking up a question in an open domain of scientific full texts and then extracting an answer from those texts as snippets of text. A nice lay person's overview of what the extractive part of this looks like is provided in <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://huggingface.co/course/chapter7/7?fw=pt" tabindex="0" role="link">Huggingface Transformers Course - Question answering</a></p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="how-is-this-different-from-other-systems">How is this different from other systems?</h4><p class="Paragraph__body____m5FY">Other than maybe some technical details presented in the next section <em>the main difference is that we are using citation statements</em> (the 3-5 sentences where a reference is used in-text) extracted from full-texts as our primary source for answering questions. We do use abstracts as well, but there is significant evidence that citation statements provide a lot of great information from summarizing the claims of an author, providing factoid answers, and surfacing criticism or supporting evidence for a particular answer. For more details on this see <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://pubmed.ncbi.nlm.nih.gov/23227044/" tabindex="0" role="link">Do peers see more in a paper than its authors?</a>. Currently we are answering questions from over 1.2bn citation statements extracted from 32 million full texts which are from both closed and open access sources as well as about 48 million abstracts.</p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="how-does-it-actually-work">How does it actually work?</h4><p class="Paragraph__body____m5FY">We currently started with a very basic approach to open-domain extractive QA.</p><p class="Paragraph__body____m5FY">The system works like this:</p><ul><li class="BlogApp__listItem___bpEyn">We process your question by removing stop words and punctuation to form a query.</li><li class="BlogApp__listItem___bpEyn">We retrieve the top 200 results from elasticsearch using that query over our 1.2bn citation statements and 48 million abstracts.</li><li class="BlogApp__listItem___bpEyn">We rerank the results with the original question using a cross-encoder trained on MS-MARCO (we use 'cross-encoder/ms-marco-MiniLM-L-12-v2' available on <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://www.sbert.net/docs/pretrained_cross-encoders.html" tabindex="0" role="link">sentence transformers</a> )</li><li class="BlogApp__listItem___bpEyn">We use a page length of 20 results and run our extractive question answering model trained on squad2, natural questions, and bioasq (see <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://huggingface.co/scite/roberta-base-squad2-nq-bioasq" tabindex="0" role="link">our model</a> for details).</li><li class="BlogApp__listItem___bpEyn">We return the answers to you!</li></ul><p class="Paragraph__body____m5FY">Our models are available here: <a class="Link__link___plGt2 Link__blueLink___BhE5_" href="https://huggingface.co/scite" tabindex="0" role="link">https://huggingface.co/scite</a></p><p class="Paragraph__body____m5FY">Specifically we use the ONNX optimized (scite/roberta-base-squad2-nq-bioasq-optimized-gpu and scite/ms-marco-MiniLM-L-12-v2-onnx-optimized-gpu)</p><p class="Paragraph__body____m5FY">We are happy to provide more details on how we built this if you are interested at customersupport@researchsolutions.com - We didn't write anything formal yet since we are still deciding on our approaches - once we have settled on an approach we will write up a paper fully describing it and releasing final assets.</p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="how-does-it-perform">How does it perform?</h4><p class="Paragraph__body____m5FY">This is actually quite an open question! We know how ms-marco-MiniLM-L-12-v2 performs on MS-MARCO (MRR@10 on MS Marco Dev Set: 39.02). We know how our model performs on SQUAD2+BIOASQ at 88.5% exact match (f1 93.3%).</p><p class="Paragraph__body____m5FY">But citation statements may be quite a different domain than these and besides individual model performance doesn't indicate how the approach performs end to end.</p><p class="Paragraph__body____m5FY">We are currently in the process for developing a benchmark for assessing performance end to end so stay tuned!</p><h4 class="Header__h4___DEzov BlogApp__header4___MrYvl" id="future-plans">Future Plans?</h4><p class="Paragraph__body____m5FY">Our future plans will largely be driven by user feedback - so far this is mostly improving the UX and UI and adding controls like filtering by year and so on. Beyond this some concrete near-term plans though are:</p><ul><li class="BlogApp__listItem___bpEyn">Develop an approach to benchmark end to end performance on our corpus</li><li class="BlogApp__listItem___bpEyn">Expand the number of datasets we use for training our QA model (We only use Squad2, Natural Questions, and BioASQ but there are so many more and even ones in the scientific domain)</li><li class="BlogApp__listItem___bpEyn">Train our reranker on in-domain ranking data (MS-MARCO is not a science specific reranker)</li><li class="BlogApp__listItem___bpEyn">Explore more approaches to query expansion (since we still ulitimately use lexical search with elastic search) or retrieval augmentation</li></ul><p class="Paragraph__body____m5FY">Some longer-term plans are to extend the extractive tasks beyond question answering and make the answers more informative for non-experts.</p><p class="Paragraph__body____m5FY">Feel free to reach out to us if you have any questions or feedback at customersupport@researchsolutions.com</p></div><div class="BlogApp__blogFooter___tIspK"><div class="ShareButtons__shareButtons___TodEt"><span>Share</span><span class="ShareButtons__icons___mhgN8"><a class="ShareButtons__icon___Ve2y5" href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__twitter___NsyXj icon icon-twitter-new"></i></a><a class="ShareButtons__icon___Ve2y5" href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__linkedin___nS70z icon icon-linkedin-new"></i></a><a class="ShareButtons__icon___Ve2y5" href="https://facebook.com/share.php?u=https%3A%2F%2Fscite.ai%2Fblog%2F2022-11-16_how_does_ask_a_question_work" tabindex="0" target="_blank" rel="noopener noreferrer"><i class="ShareButtons__facebook___X8lcc icon icon-facebook-new"></i></a></span></div></div></div></div></div><div class="Footer__layout___jrAhC"><div class="Footer__mainLayout___MVOfi"><div class="Footer__aboutContainer___TwE0S"><img width="128px" height="41px" alt="scite logo" src="https://cdn.scite.ai/assets/images/logo.svg"/><p class="Paragraph__body____m5FY">scite is a Brooklyn-based organization that helps researchers better discover and understand research articles through Smart Citations–citations that display the context of the citation and describe whether the article provides supporting or contrasting evidence. scite is used by students and researchers from around the world and is funded in part by the National Science Foundation and the National Institute on Drug Abuse of the National Institutes of Health.</p><div class="Footer__contact___DGc_M"><h6 class="Header__h6___QzBvp">Contact Info</h6><div><p class="Footer__address___VMUTY Paragraph__body____m5FY">customersupport@researchsolutions.com</p><p class="Footer__address___VMUTY Paragraph__body____m5FY">10624 S. Eastern Ave., Ste. A-614</p><p class="Footer__address___VMUTY Paragraph__body____m5FY">Henderson, NV 89052, USA</p></div></div></div><div class="Footer__productContainer___wG2mf"><div><h6 class="Header__h6___QzBvp">Product</h6><div class="Footer__products___wIN0z"><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/extension-install">Browser Extension</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/assistant">Assistant by scite</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/search/citations">Citation Statement Search</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/?referenceCheck=true">Reference Check</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/visualizations/global-analysis-of-genome-transcriptome-9L4dJr?dois[0]=10.1038%2Fmsb.2012.40&dois[1]=10.7554%2Felife.05068&focusedElement=10.7554%2Felife.05068">Visualizations</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/create-dashboard">Dashboards</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/journals">Explore Journals</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/affiliations">Explore Organizations</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/institutions">Explore Funders</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/badge">Embedding Badge</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/embed-citation-search">Embedding Citation Search</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/pricing">Pricing</a></div></div><div><h6 class="Header__h6___QzBvp">Resources</h6><div class="Footer__resources___N8pen"><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" href="https://scite.ai/blog" tabindex="0" role="link" target="_blank" rel="noopener noreferrer">Blog</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" href="https://help.researchsolutions.com/hc/en-us/categories/31947397460116-Scite" tabindex="0" role="link" target="_blank" rel="noopener noreferrer">Help & FAQ</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/accessibility-statement">Accessibility Statement</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/apiterms">API Terms</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/partners/academic-institutions">For Universities & Governments</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/partners/researchers">For Researchers</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/partners/publishers">For Publishers</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/partners/pharmaceuticals-and-biotech">For Corporate, Pharma & Enterprise</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/author-marketing">Author Marketing</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" target="_blank" rel="noopener noreferrer" href="/affiliate">Become an Affiliate</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/pricing?tab=organizations">Get an organization trial or quote</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/data-and-services">scite Data & Services</a></div></div><div><h6 class="Header__h6___QzBvp">About</h6><div class="Footer__about___guUpd"><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" target="_blank" rel="noopener noreferrer" href="/news-and-press">News & Press</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/jobs">Careers</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" href="https://direct.mit.edu/qss/article/2/3/882/102990/scite-A-smart-citation-index-that-displays-the" tabindex="0" role="link">Read our Paper</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/data-and-services">Coverage</a></div></div></div></div><div class="Footer__recaptchaInfo___uzydh"><p class="Paragraph__body____m5FY">This site is protected by reCAPTCHA and the Google <a class="Link__link___plGt2 Link__underline___K1Y7j" href="https://policies.google.com/privacy" tabindex="0" role="link" target="_blank" rel="noopener noreferrer">Privacy Policy</a> and <a class="Link__link___plGt2 Link__underline___K1Y7j" href="https://policies.google.com/terms" tabindex="0" role="link" target="_blank" rel="noopener noreferrer">Terms of Service</a> apply.</p></div><div class="Footer__staticContainer___STnbm"><div class="Footer__staticLayout___KI3Dv"><p class="Footer__staticLinks___luYcW Paragraph__small___OZxzF"><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" href="https://scite.ai/blog" tabindex="0" role="link" target="_blank" rel="noopener noreferrer">Blog</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/terms">Terms and Conditions</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/apiterms">API Terms</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" href="/policy">Privacy Policy</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" href="mailto:customersupport@researchsolutions.com" tabindex="0" role="link">Contact</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" role="button">Cookie Preferences</a><a class="Footer__link____xReJ Link__link___plGt2 Footer__link____xReJ" tabindex="0" role="button">Do Not Sell or Share My Personal Information</a></p><div class="Footer__iconsLayout___av51Y"><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://twitter.com/scite" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://twitter.com/scite"><i class="icon icon-twitter-new"></i></a><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://facebook.com/sciteai" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://facebook.com/sciteai"><i class="icon icon-facebook-new"></i></a><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://www.instagram.com/scite.ai/" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://www.instagram.com/scite.ai/"><i class="icon icon-instagram-new"></i></a><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://www.tiktok.com/@scite.ai" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://www.tiktok.com/@scite.ai"><i class="icon icon-tiktok"></i></a><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://www.linkedin.com/company/sciteai/" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://www.linkedin.com/company/sciteai/"><i class="icon icon-linkedin-new"></i></a><a class="SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr Link__link___plGt2 SocialIcon__socialIcon___gm1fE Footer__icon___m4hTr" href="https://www.youtube.com/channel/UCso9-KdHZ6vfbYgM59zp9EA" tabindex="0" role="link" target="_blank" rel="noopener noreferrer" aria-label="https://www.youtube.com/channel/UCso9-KdHZ6vfbYgM59zp9EA"><i class="icon icon-youtube-new"></i></a></div></div></div><div class="Footer__copyrightContainer____MV8W"><div class="Footer__copyrightLayout___XD0J7"><p class="Paragraph__small___OZxzF">Copyright © 2024 scite LLC. All rights reserved.</p><p class="Paragraph__small___OZxzF">Made with 💙 for researchers</p><div class="Footer__rsssFamily___jPDL1"><p class="Footer__rsssBlurb___XhmEW Paragraph__small___OZxzF">Part of the Research Solutions Family.</p><img src="https://cdn.scite.ai/assets/images/logo-rs-black-new.png" class="Footer__rsssLogo___bG803"/></div></div></div></div><div style="position:fixed;z-index:9999"></div></div><div id="scite-tooltips"></div><script type="application/javascript">window.__SCITE_STORE_STATE = {"addReportDashboardModal":{"selectedPapers":[]},"ui":{"referralPartner":null,"freeReports":0,"bannerContent":null,"bannerClosed":false,"navMenu":{},"createDashboardFromSearchResultsModalShow":false,"savedSearchModalShow":false,"searchFilterModalShow":false,"flagModalShow":false,"resetPasswordModalShow":false,"loginModalShow":false,"referenceCheckModalShow":false,"deleteDashboardModalShow":false,"addReportToDashboardModalShow":false,"shareDashboardModalShow":false,"editDashboardModalShow":false,"upgradeModalShow":false,"exportReportModalShow":false,"exportReferenceModalShow":false,"generateCitationModalShow":false,"exportDashboardModalShow":false,"dashboardLimitErrorModalShow":false,"whatIsThisReportModalShow":false,"whatIsThisSearchModalShow":false,"whatIsAssistantModalShow":false,"trialOverModalShow":false,"activateTrialModalShow":false,"cancelTrialCheckoutModalShow":false,"cookieModalShow":false,"personaModalShow":false,"loggedOutModalShow":true,"modalFromContext":"","shownModals":[],"hasUser":false,"user":null,"ipLocationData":{"data":null,"error":false,"loading":false},"fetchingUser":false,"savingUser":false,"saveUserError":false,"onboardingLoadTracker":{"state":undefined},"userEntitlements":null,"entitlementsLoading":false,"referralsLoading":false,"pageError":null,"browser":null,"extensionLoaded":false,"organizationLicense":{"hasLicense":false},"linkResolver":null,"sickoMode":false,"authors":[],"navOpen":false,"showMobileSearch":false,"redeemCodeLoading":false,"redeemCodeError":null,"redeemCodeSuccess":false,"applyDiscountLoading":false,"applyDiscountError":null,"applyDiscountSuccess":false,"userDiscount":null,"search":{"term":"","focused":false,"autocompletesLoadTracker":{},"autocompletes":{}},"referencesToExport":[],"citeGeneratorPaperData":{},"dimensions":{},"cookiePreference":null,"userChurnKey":null,"churnKeyLoading":false,"fullTextLinkState":{},"agAvailabilityState":{},"agAvailabilityQueue":[]},"verifyEmail":{"loading":false,"verified":false,"expired":false},"dashboard":{"id":"","name":"","displayName":"","normalizedName":"","description":"","slug":"","dashboardType":"","accessType":"","isPublic":null,"componentLayout":[],"doiQuery":{},"journalMetadata":{},"affiliationMetadata":{},"institutionMetadata":{},"haveAccessToDashboard":null,"lastUpdated":null,"doisUpdatedTS":null,"doisRefreshedTS":null,"refreshStatus":"","loadState":{},"deleteDashboardError":false,"deletingDashboard":false,"totalDoiCount":0,"matchedDoiCount":0,"unmatchedDoiCount":0,"exportLoadTracker":{},"setDashboardUsersLoading":false,"setDashboardUsersError":false,"setDashboardUsersSuccess":false,"dashboardAccessList":null,"getDashboardAccessListLoading":false,"getDashboardAccessListSuccess":false,"getDashboardAccessListError":false},"notifications":{"notifications":[],"loading":false,"error":null,"notificationItems":[],"notificationItemsLoading":false,"notificationItemsError":null,"notificationItemsPage":1,"notificationItemsTotal":0,"newNotificationItems":0,"notificationItemsFilters":{},"dropDownNotificationItems":[],"dropDownNotificationItemsLoading":false,"dropDownNotificationItemsError":null,"notificationSetStatusLoading":false,"notificationSetStatusError":null},"blog":{"post":{"data":{"title":"How does ask a question work?","subheader":"Ask a question is an extractive question answering system on over 1.2bn citation statements.","description":"Ask a question helps you ask simple, natural questions and get answers extracted from more than 32 million full texts (closed and open access). How does it work?","author":"scite","date":"16 November 2022","categories":["Research"],"slug":"2022-11-16_how_does_ask_a_question_work"},"excerpt":"","content":"\nRecently we released an early preview of a question answering (QA) system built on top of our data (see [https:\u002F\u002Fscite.ai\u002Fdata-and-services](https:\u002F\u002Fscite.ai\u002Fdata-and-services) for a more comprehensive picture of what our data is). This short article will explain how it *currently* works as of November 16th 2022 and a little bit about our current plans for expanding it\n\nThis is a fairly technical post so feel free to reach out to us at customersupport@researchsolutions.com if you have questions.\n\n#### Why build a scientific question answering system?\n\nWhile we have a pretty powerful search system that already allows you to search 1.2bn citation statements extracted from 32 million full texts (both closed and open access!)... search isn't always the easiest way to get after the kinds of information you are looking for. This is especially true since we use a \"lexical\" (read word matching)-based search system using elasticsearch - so if you don't formulate your search query exactly with the words you want matched you might not get what you are looking for. In addition, much of the \"information seeking\" experiences we have when looking at science comes in the form of questions we need answers to. Therefore, we thought it might help our users discover science-backed answers if we build a QA system over all our data!\n\n**We are building a large-scale scientific question answering system to help users get science-based answers to questions**.\n\n#### What is question answering?\n\nQuestion answering is a popular task in the natural language processing community that seeks to provide an answer for a given question posed by the user. There are plenty of formulations of question answering from multiple choice, closed book, and extractive qa to open domain and multi-hop qa. It is a very popular task; for a great survey on the recent explosion of datasets alone see [QA Dataset Explosion: A Taxonomy of NLP Resources for Question Answering and Reading Comprehension](https:\u002F\u002Farxiv.org\u002Fabs\u002F2107.12708).\n\nOur approach is *open-domain extractive question answering.* This means that we are looking up a question in an open domain of scientific full texts and then extracting an answer from those texts as snippets of text. A nice lay person's overview of what the extractive part of this looks like is provided in [Huggingface Transformers Course - Question answering](https:\u002F\u002Fhuggingface.co\u002Fcourse\u002Fchapter7\u002F7?fw=pt)\n\n#### How is this different from other systems?\n\nOther than maybe some technical details presented in the next section *the main difference is that we are using citation statements* (the 3-5 sentences where a reference is used in-text) extracted from full-texts as our primary source for answering questions. We do use abstracts as well, but there is significant evidence that citation statements provide a lot of great information from summarizing the claims of an author, providing factoid answers, and surfacing criticism or supporting evidence for a particular answer. For more details on this see [Do peers see more in a paper than its authors?](https:\u002F\u002Fpubmed.ncbi.nlm.nih.gov\u002F23227044\u002F). Currently we are answering questions from over 1.2bn citation statements extracted from 32 million full texts which are from both closed and open access sources as well as about 48 million abstracts.\n\n#### How does it actually work?\n\nWe currently started with a very basic approach to open-domain extractive QA.\n\nThe system works like this:\n * We process your question by removing stop words and punctuation to form a query.\n * We retrieve the top 200 results from elasticsearch using that query over our 1.2bn citation statements and 48 million abstracts.\n * We rerank the results with the original question using a cross-encoder trained on MS-MARCO (we use 'cross-encoder\u002Fms-marco-MiniLM-L-12-v2' available on [sentence transformers](https:\u002F\u002Fwww.sbert.net\u002Fdocs\u002Fpretrained_cross-encoders.html) )\n * We use a page length of 20 results and run our extractive question answering model trained on squad2, natural questions, and bioasq (see [our model](https:\u002F\u002Fhuggingface.co\u002Fscite\u002Froberta-base-squad2-nq-bioasq) for details).\n * We return the answers to you!\n\nOur models are available here: https:\u002F\u002Fhuggingface.co\u002Fscite\n\nSpecifically we use the ONNX optimized (scite\u002Froberta-base-squad2-nq-bioasq-optimized-gpu and scite\u002Fms-marco-MiniLM-L-12-v2-onnx-optimized-gpu)\n\nWe are happy to provide more details on how we built this if you are interested at customersupport@researchsolutions.com - We didn't write anything formal yet since we are still deciding on our approaches - once we have settled on an approach we will write up a paper fully describing it and releasing final assets.\n\n#### How does it perform?\n\nThis is actually quite an open question!\nWe know how ms-marco-MiniLM-L-12-v2 performs on MS-MARCO (MRR@10 on MS Marco Dev Set: 39.02).\nWe know how our model performs on SQUAD2+BIOASQ at 88.5% exact match (f1 93.3%).\n\nBut citation statements may be quite a different domain than these and besides individual model performance doesn't indicate how the approach performs end to end.\n\nWe are currently in the process for developing a benchmark for assessing performance end to end so stay tuned!\n\n#### Future Plans?\n\nOur future plans will largely be driven by user feedback - so far this is mostly improving the UX and UI and adding controls like filtering by year and so on.\nBeyond this some concrete near-term plans though are:\n- Develop an approach to benchmark end to end performance on our corpus\n- Expand the number of datasets we use for training our QA model (We only use Squad2, Natural Questions, and BioASQ but there are so many more and even ones in the scientific domain)\n- Train our reranker on in-domain ranking data (MS-MARCO is not a science specific reranker)\n- Explore more approaches to query expansion (since we still ulitimately use lexical search with elastic search) or retrieval augmentation\n\nSome longer-term plans are to extend the extractive tasks beyond question answering and make the answers more informative for non-experts.\n\nFeel free to reach out to us if you have any questions or feedback at customersupport@researchsolutions.com\n"},"loadingPost":false,"indexPosts":[{"title":"Major Updates to scite Assistant: A Fresh New Look and More","date":"24 April 2024","categories":["Other"],"keywords":[],"author":"scite","slug":"major_updates_to_scite_assistant","description":"It's been over a year since we first launched scite Assistant, and we've been hard at work making improvements based on the valuable feedback from our users. We recently unveiled a brand new look for scite Assistant along with several enhancements to improve your experience."},{"title":"Using scite Assistant in the classroom: Locating measures in the behavioral sciences","date":"24 January 2024","categories":["Other"],"keywords":[],"author":"scite","slug":"locating-measures-behavioral-sciences","description":"As a psychology professor, I regularly teach research methods courses. Over a two-course sequence, I teach my students various approaches to empirical psychological research. This culminates in a semester-long project in which students formulate a hypothesis, design a study, collect and analyze data, and communicate the results."},{"title":"A brief overview of Shepard’s Citations, Bibliometrics, and Smart Citations","date":"8 January 2024","categories":["Other"],"keywords":[],"author":"scite","slug":"brief-overview-Shepards-Citations","description":"Bibliometrics is a field of study that involves the quantitative analysis of scholarly publications and their citations to evaluate various aspects of research, such as impact, productivity, and collaboration. The history of bibliometrics can be traced back to the pioneering work of Eugene Garfield, who introduced the first citation index for sciences in 1953 (Masic, 2016). "},{"title":"On Joining Research Solutions","date":"27 November 2023","categories":["News"],"keywords":[],"author":"scite","slug":"on-joining-research-solutions","description":"We’re excited to enter a new phase of our journey at scite by coming together with [Research Solutions](https:\u002F\u002Fwww.researchsolutions.com\u002F). The motivation behind this decision was our desire to further our mission of introducing the next generation of citations and better serve our users, publishers, and the scholarly ecosystem for the long term."},{"title":"Using scite to speed up literature reviews and critical analysis 🚀","date":"22 March 2023","categories":["Tutorial"],"keywords":[],"author":"scite","slug":"recite_using-scite-to-speed-up-literature-reviews-and-critical-analysis","description":"Literature reviews and critical analyses take time. In order to write high quality research you need to engage with the literature, read the references being cited, read the citations to the paper and make sure you understand the field as a whole. Collecting and reading all relevant references and citations takes a long time. scite helps you **spending less time on research and writing** while simultaneously **boosting the quality of your work** by presenting what is said about a paper right away."},{"title":"Colabra and scite Partner on the Development of Citations Block","author":"scite","date":"27 February 2023","description":"Brooklyn, NY — February 23, 2023 — scite, an award-winning tool that helps researchers discover and understand research findings more efficiently through Smart Citations, has partnered with Colabra, provider of a modern GxP-compliant Electronic Laboratory Notebook (ELN) and project management tool, to enhance the citation experience on the Colabra platform.","categories":["News"],"slug":"2023-02-27_colabra-and-scite-partner-on-the-development-of-citations-block"},{"title":"How might scite impact early career researchers?","author":"scite","date":"23 February 2023","categories":["Other"],"slug":"2023-02-23_ECRs","description":"One question that we sometimes get is how scite might impact early career researchers (ECRs). For example, many metrics (e.g., the h-index) have a built-in bias against ECRs, as those in a given field for longer periods have more time to accumulate more citations, and in turn increase their score."},{"title":"scite's coverage of the scholarly literature","author":"scite","date":"16 February 2023","categories":["Other"],"slug":"2023-02-16_coverage","description":"One question that often comes up relates to coverage: that is, how many articles are indexed in scite, and how many do we have citations from? This question is sometimes posed as a comparison to other services such as Scopus, Web of Science, and PubMed."},{"title":"Using scite to find active research topics","author":"scite","date":"16 January 2023","categories":["Tutorial"],"slug":"2023-01-16_active_topics","description":"One of my favorite pastimes is using scite to find active areas of research - topics that invite lively discussion, or hypotheses that are highly contested. This can be accomplished by searching for a topic and filtering the results to only show articles that have a relatively high number of both supporting and contrasting citations."},{"title":"What does science say about New Year's resolutions?","author":"scite","date":"31 December 2022","categories":["Other"],"slug":"2022-12-31_new_years_resolutions","description":"Is the new year time to make a new you? Let’s use scite to see what science says about New Year's resolutions."},{"title":"What does science say about the holidays?","author":"scite","date":"23 December 2022","categories":["Other"],"slug":"2022-12-23_what_does_science_say_about_holidays","description":"As the holiday season is now upon us, let’s have a little fun and use scite’s new “Ask a Question” feature to find out what the scientific literature says about this time of year."},{"title":"What does the research say about bullshit?","author":"scite","date":"8 December 2022","categories":["Other"],"slug":"2022-12-08_what_does_science_say_about_bs","description":"As a researcher and really just as a human nowdays, we’re surrounded by a lot of claims, conjectures, opinions, and really just bullshit. As a psychologist, this got me thinking… What does the research say about bullshit and bullshitting. So, using [Ask A Question](https:\u002F\u002Fscite.ai\u002Fsearch?mode=question-answering), I asked:"},{"title":"SPIE and scite Partner on the development of Smart Citations","subheader":"scite has partnered with SPIE, the international society for optics and photonics, to enhance the research experience on the SPIE Digital Library.","description":"scite has partnered with SPIE, the international society for optics and photonics, to enhance the research experience on the SPIE Digital Library.","author":"scite","date":"23 November 2022","categories":["News"],"slug":"2022-11-23_spie-and-scite"},{"title":"A Global Database of Citation Context and Coverage: Our Coverage in Turkey","subheader":"Learn how scite makes research more inclusive and comprehensive by including Turkish content in its database.","description":"Learn how scite makes research more inclusive and comprehensive by including Turkish content in its database.","author":"scite","date":"21 November 2022","categories":["News"],"slug":"2022-11-21_turkish_citation_coverage"},{"title":"Ask a Question, Get Answers Directly from Research Article","subheader":"Today, we're excited to introduce the capability for scite users to ask research questions in plain language and get answers directly from the full text of research articles.","description":"Today, we're excited to introduce the capability for scite users to ask research questions in plain language and get answers directly from the full text of research articles.","author":"scite","date":"16 November 2022","categories":["News","New feature"],"slug":"2022-11-16_ask_a_question_get_answers_directly"},{"title":"How does ask a question work?","subheader":"Ask a question is an extractive question answering system on over 1.2bn citation statements.","description":"Ask a question helps you ask simple, natural questions and get answers extracted from more than 32 million full texts (closed and open access). How does it work?","author":"scite","date":"16 November 2022","categories":["Research"],"slug":"2022-11-16_how_does_ask_a_question_work"},{"title":"scite Recruits Rogier van Erkel as Chief Revenue Officer","subheader":"Rogier brings significant experience leading sales in academia and corporate markets.","description":"Rogier brings significant experience leading sales in academia and corporate markets.","author":"scite","date":"15 November 2022","categories":["News"],"slug":"2022-11-15_welcome-rogier"},{"title":"Igor Osipov joins scite as VP of Global Institutional Markets","subheader":"scite grows its team with Igor Osipov leading engagements in global institutional markets.","description":"scite grows its team with Igor Osipov leading engagements in global institutional markets.","author":"scite","date":"09 November 2022","categories":["News"],"slug":"2022-11-09_welcome-igor"},{"title":"A new, open system for the automated detection of retractions and other editorial notices in the scholarly ecosystem","subheader":"See how we're making editorial notices, including retractions, more transparent for everyone.","description":"See how we're making editorial notices, including retractions, more transparent for everyone.","author":"scite","date":"19 October 2022","categories":["News"],"slug":"2022-10-19_automated-notice-detection"},{"title":"Lighting up the Scientific Article: How Citation Statements Can Turn Key Topics in Articles into Rich Sources of Information","subheader":"Learn how our Citation Statements can help readers get a qualitative understanding of any topic.","description":"Learn how our Citation Statements can help readers get a qualitative understanding of any topic.","author":"scite","date":"21 September 2022","categories":["News"],"slug":"2022-09-21_lighting-up-scientific-article"}],"totalPosts":117,"category":"","categories":["case study","development","new feature","news","other","research","tutorial"],"indexPage":1,"indexLoadTracker":{"state":"loaded","key":"1"}}}</script><script id="__LOADABLE_REQUIRED_CHUNKS__" type="application/json">[]</script><script id="__LOADABLE_REQUIRED_CHUNKS___ext" type="application/json">{"namedChunks":[]}</script><script async="" data-chunk="main" src="https://cdn.scite.ai/assets/dist/main.33b6f5c40898cc5cded7.js"></script><div><script id="profitwell-js" data-pw-auth="eba7f4438ac9b91fc798c9620e35aa80"> (function(i,s,o,g,r,a,m){i[o]=i[o]||function(){(i[o].q=i[o].q||[]).push(arguments)}; a=s.createElement(g);m=s.getElementsByTagName(g)[0];a.async=1;a.src=r+'?auth='+ s.getElementById(o+'-js').getAttribute('data-pw-auth');m.parentNode.insertBefore(a,m); })(window,document,'profitwell','script','https://public.profitwell.com/js/profitwell.js'); </script> <script async src="https://www.googletagmanager.com/gtag/js?id=AW-614125746"></script> <script> window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag("consent", "default", {ad_storage: "denied", analytics_storage: "denied", wait_for_update: 1500}); gtag('js', new Date()); gtag('config', 'AW-614125746'); </script> <script src="https://www.dwin1.com/19038.js" type="text/javascript" defer="defer"></script> <script src="https://script.tapfiliate.com/tapfiliate.js" type="text/javascript" async></script> <script type="text/javascript"> (function(t,a,p){t.TapfiliateObject=a;t[a]=t[a]||function(){ (t[a].q=t[a].q||[]).push(arguments)}})(window,'tap'); tap('create', '48731-1a14bc', { integration: "javascript" }); tap('detect'); </script> <script src="https://fast.wistia.com/embed/medias/o5vno207sz.jsonp" async></script> <script src="https://fast.wistia.com/assets/external/E-v1.js" async></script>  <script> window.__MathJax_State__ = { isReady: false, promise: new Promise(resolve => { window.MathJax = { // MathJax can be configured as desired in addition to these options. startup: { ready: () => { // Do whatever MathJax would normally do at this point. MathJax.startup.defaultReady(); // Set the flag and resolve the promise. window.__MathJax_State__.isReady = true; resolve(); } }, tex: { inlineMath: [ ["$", "$"], ], }, svg: { fontCache: "global" } }; }) }; </script> <script> var _rollbarConfig = { accessToken: "2c3f1a7fb80c4667a8370b6d5e60b850", captureUncaught: true, captureUnhandledRejections: true, payload: { environment: "production", client: { javascript: { code_version: "null" } } } }; // Rollbar Snippet !function(r){function e(n){if(o[n])return o[n].exports;var t=o[n]={exports:{},id:n,loaded:!1};return r[n].call(t.exports,t,t.exports,e),t.loaded=!0,t.exports}var o={};return e.m=r,e.c=o,e.p="",e(0)}([function(r,e,o){"use strict";var n=o(1),t=o(4);_rollbarConfig=_rollbarConfig||{},_rollbarConfig.rollbarJsUrl=_rollbarConfig.rollbarJsUrl||"https://cdnjs.cloudflare.com/ajax/libs/rollbar.js/2.24.0/rollbar.min.js",_rollbarConfig.async=void 0===_rollbarConfig.async||_rollbarConfig.async;var a=n.setupShim(window,_rollbarConfig),l=t(_rollbarConfig);window.rollbar=n.Rollbar,a.loadFull(window,document,!_rollbarConfig.async,_rollbarConfig,l)},function(r,e,o){"use strict";function n(r){return function(){try{return r.apply(this,arguments)}catch(r){try{console.error("[Rollbar]: Internal error",r)}catch(r){}}}}function t(r,e){this.options=r,this._rollbarOldOnError=null;var o=s++;this.shimId=function(){return o},"undefined"!=typeof window&&window._rollbarShims&&(window._rollbarShims[o]={handler:e,messages:[]})}function a(r,e){if(r){var o=e.globalAlias||"Rollbar";if("object"==typeof r[o])return r[o];r._rollbarShims={},r._rollbarWrappedError=null;var t=new p(e);return n(function(){e.captureUncaught&&(t._rollbarOldOnError=r.onerror,i.captureUncaughtExceptions(r,t,!0),i.wrapGlobals(r,t,!0)),e.captureUnhandledRejections&&i.captureUnhandledRejections(r,t,!0);var n=e.autoInstrument;return e.enabled!==!1&&(void 0===n||n===!0||"object"==typeof n&&n.network)&&r.addEventListener&&(r.addEventListener("load",t.captureLoad.bind(t)),r.addEventListener("DOMContentLoaded",t.captureDomContentLoaded.bind(t))),r[o]=t,t})()}}function l(r){return n(function(){var e=this,o=Array.prototype.slice.call(arguments,0),n={shim:e,method:r,args:o,ts:new Date};window._rollbarShims[this.shimId()].messages.push(n)})}var i=o(2),s=0,d=o(3),c=function(r,e){return new t(r,e)},p=d.bind(null,c);t.prototype.loadFull=function(r,e,o,t,a){var l=function(){var e;if(void 0===r._rollbarDidLoad){e=new Error("rollbar.js did not load");for(var o,n,t,l,i=0;o=r._rollbarShims[i++];)for(o=o.messages||[];n=o.shift();)for(t=n.args||[],i=0;i<t.length;++i)if(l=t[i],"function"==typeof l){l(e);break}}"function"==typeof a&&a(e)},i=!1,s=e.createElement("script"),d=e.getElementsByTagName("script")[0],c=d.parentNode;s.crossOrigin="",s.src=t.rollbarJsUrl,o||(s.async=!0),s.onload=s.onreadystatechange=n(function(){if(!(i||this.readyState&&"loaded"!==this.readyState&&"complete"!==this.readyState)){s.onload=s.onreadystatechange=null;try{c.removeChild(s)}catch(r){}i=!0,l()}}),c.insertBefore(s,d)},t.prototype.wrap=function(r,e,o){try{var n;if(n="function"==typeof e?e:function(){return e||{}},"function"!=typeof r)return r;if(r._isWrap)return r;if(!r._rollbar_wrapped&&(r._rollbar_wrapped=function(){o&&"function"==typeof o&&o.apply(this,arguments);try{return r.apply(this,arguments)}catch(o){var e=o;throw e&&("string"==typeof e&&(e=new String(e)),e._rollbarContext=n()||{},e._rollbarContext._wrappedSource=r.toString(),window._rollbarWrappedError=e),e}},r._rollbar_wrapped._isWrap=!0,r.hasOwnProperty))for(var t in r)r.hasOwnProperty(t)&&(r._rollbar_wrapped[t]=r[t]);return r._rollbar_wrapped}catch(e){return r}};for(var u="log,debug,info,warn,warning,error,critical,global,configure,handleUncaughtException,handleUnhandledRejection,captureEvent,captureDomContentLoaded,captureLoad".split(","),f=0;f<u.length;++f)t.prototype[u[f]]=l(u[f]);r.exports={setupShim:a,Rollbar:p}},function(r,e){"use strict";function o(r,e,o){if(r){var t;"function"==typeof e._rollbarOldOnError?t=e._rollbarOldOnError:r.onerror&&!r.onerror.belongsToShim&&(t=r.onerror,e._rollbarOldOnError=t);var a=function(){var o=Array.prototype.slice.call(arguments,0);n(r,e,t,o)};a.belongsToShim=o,r.onerror=a}}function n(r,e,o,n){r._rollbarWrappedError&&(n[4]||(n[4]=r._rollbarWrappedError),n[5]||(n[5]=r._rollbarWrappedError._rollbarContext),r._rollbarWrappedError=null),e.handleUncaughtException.apply(e,n),o&&o.apply(r,n)}function t(r,e,o){if(r){"function"==typeof r._rollbarURH&&r._rollbarURH.belongsToShim&&r.removeEventListener("unhandledrejection",r._rollbarURH);var n=function(r){var o,n,t;try{o=r.reason}catch(r){o=void 0}try{n=r.promise}catch(r){n="[unhandledrejection] error getting `promise` from event"}try{t=r.detail,!o&&t&&(o=t.reason,n=t.promise)}catch(r){t="[unhandledrejection] error getting `detail` from event"}o||(o="[unhandledrejection] error getting `reason` from event"),e&&e.handleUnhandledRejection&&e.handleUnhandledRejection(o,n)};n.belongsToShim=o,r._rollbarURH=n,r.addEventListener("unhandledrejection",n)}}function a(r,e,o){if(r){var n,t,a="EventTarget,Window,Node,ApplicationCache,AudioTrackList,ChannelMergerNode,CryptoOperation,EventSource,FileReader,HTMLUnknownElement,IDBDatabase,IDBRequest,IDBTransaction,KeyOperation,MediaController,MessagePort,ModalWindow,Notification,SVGElementInstance,Screen,TextTrack,TextTrackCue,TextTrackList,WebSocket,WebSocketWorker,Worker,XMLHttpRequest,XMLHttpRequestEventTarget,XMLHttpRequestUpload".split(",");for(n=0;n<a.length;++n)t=a[n],r[t]&&r[t].prototype&&l(e,r[t].prototype,o)}}function l(r,e,o){if(e.hasOwnProperty&&e.hasOwnProperty("addEventListener")){for(var n=e.addEventListener;n._rollbarOldAdd&&n.belongsToShim;)n=n._rollbarOldAdd;var t=function(e,o,t){n.call(this,e,r.wrap(o),t)};t._rollbarOldAdd=n,t.belongsToShim=o,e.addEventListener=t;for(var a=e.removeEventListener;a._rollbarOldRemove&&a.belongsToShim;)a=a._rollbarOldRemove;var l=function(r,e,o){a.call(this,r,e&&e._rollbar_wrapped||e,o)};l._rollbarOldRemove=a,l.belongsToShim=o,e.removeEventListener=l}}r.exports={captureUncaughtExceptions:o,captureUnhandledRejections:t,wrapGlobals:a}},function(r,e){"use strict";function o(r,e){this.impl=r(e,this),this.options=e,n(o.prototype)}function n(r){for(var e=function(r){return function(){var e=Array.prototype.slice.call(arguments,0);if(this.impl[r])return this.impl[r].apply(this.impl,e)}},o="log,debug,info,warn,warning,error,critical,global,configure,handleUncaughtException,handleUnhandledRejection,_createItem,wrap,loadFull,shimId,captureEvent,captureDomContentLoaded,captureLoad".split(","),n=0;n<o.length;n++)r[o[n]]=e(o[n])}o.prototype._swapAndProcessMessages=function(r,e){this.impl=r(this.options);for(var o,n,t;o=e.shift();)n=o.method,t=o.args,this[n]&&"function"==typeof this[n]&&("captureDomContentLoaded"===n||"captureLoad"===n?this[n].apply(this,[t[0],o.ts]):this[n].apply(this,t));return this},r.exports=o},function(r,e){"use strict";r.exports=function(r){return function(e){if(!e&&!window._rollbarInitialized){r=r||{};for(var o,n,t=r.globalAlias||"Rollbar",a=window.rollbar,l=function(r){return new a(r)},i=0;o=window._rollbarShims[i++];)n||(n=o.handler),o.handler._swapAndProcessMessages(l,o.messages);window[t]=n,window._rollbarInitialized=!0}}}}]); // End Rollbar Snippet </script> <script> (function(w,d,t,r,u) { var f,n,i; w[u]=w[u]||[],f=function() { var o={ti:"343050743", enableAutoSpaTracking: true}; o.q=w[u],w[u]=new UET(o),w[u].push("pageLoad") }, n=d.createElement(t),n.src=r,n.async=1,n.onload=n.onreadystatechange=function() { var s=this.readyState; s&&s!=="loaded"&&s!=="complete"||(f(),n.onload=n.onreadystatechange=null) }, i=d.getElementsByTagName(t)[0],i.parentNode.insertBefore(n,i) }) (window,document,"script","//bat.bing.com/bat.js","uetq"); </script> <script> !function(f,b,e,v,n,t,s) {if(f.fbq)return;n=f.fbq=function(){n.callMethod? n.callMethod.apply(n,arguments):n.queue.push(arguments)}; if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0'; n.queue=[];t=b.createElement(e);t.async=!0; t.src=v;s=b.getElementsByTagName(e)[0]; s.parentNode.insertBefore(t,s)}(window, document,'script', 'https://connect.facebook.net/en_US/fbevents.js'); fbq('init', '927411631440706'); fbq('track', 'PageView'); </script> <noscript><img height="1" width="1" style="display:none" src="https://www.facebook.com/tr?id=927411631440706&ev=PageView&noscript=1" /></noscript> <script> !function(){ if (!window.churnkey || !window.churnkey.created) { window.churnkey = { created: true }; const a = document.createElement('script'); a.src = 'https://assets.churnkey.co/js/app.js?appId=2l1x8e2dl'; a.async = true; const b = document.getElementsByTagName('script')[0]; b.parentNode.insertBefore(a, b); } }(); </script> <script src='https://js.chilipiper.com/marketing.js' type='text/javascript' /> <script src="https://tag.clearbitscripts.com/v1/pk_fc279c8e6fe02304f4b6e672bc239206/tags.js" referrerpolicy="strict-origin-when-cross-origin"></script> </div></body>