Unmasking Hypnotized AI: The Hidden Risks of Large

<!doctype html> <html lang="en-US"> <head> <meta charset="UTF-8"> <link rel="shortcut icon" type="image/x-icon" href="https://securityintelligence.com/wp-content/themes/sapphire/images/favicon.ico" sizes="32x32" /> <meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1,maximum-scale=1">  <title>Unmasking Hypnotized AI: The Hidden Risks of Large</title>   <meta name="theme-color" content="#000000">  <meta name="referrer" content="no-referrer-when-downgrade">   <script async src="https://cdn.ampproject.org/v0.js"></script> <script async custom-element="amp-list" src="https://cdn.ampproject.org/v0/amp-list-0.1.js"></script> <script async custom-template="amp-mustache" src="https://cdn.ampproject.org/v0/amp-mustache-0.2.js"></script> <script async custom-element="amp-accordion" src="https://cdn.ampproject.org/v0/amp-accordion-0.1.js"></script> <script custom-element="amp-animation" src="https://cdn.ampproject.org/v0/amp-animation-0.1.js" async></script> <script custom-element="amp-position-observer" src="https://cdn.ampproject.org/v0/amp-position-observer-0.1.js" async></script> <script async custom-element="amp-bind" src="https://cdn.ampproject.org/v0/amp-bind-0.1.js"></script> <script async custom-element="amp-autocomplete" src="https://cdn.ampproject.org/v0/amp-autocomplete-0.1.js"></script> <script async custom-element="amp-social-share" src="https://cdn.ampproject.org/v0/amp-social-share-0.1.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/version/v1.35.0/card-section-simple.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/tag/v1/latest/card-section-simple.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/tag/v1/next/card-section-simple.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/version/v2.11.0/card.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/version/v2.11.0/image.min.js"></script> <script async custom-element="amp-lightbox-gallery" src="https://cdn.ampproject.org/v0/amp-lightbox-gallery-0.1.js"></script> <script src="https://unpkg.com/swiper/swiper-bundle.min.js"></script> <script async custom-element="amp-video" src="https://cdn.ampproject.org/v0/amp-video-0.1.js"></script> <script async custom-element="amp-youtube" src="https://cdn.ampproject.org/v0/amp-youtube-0.1.js"></script> <link rel="preload" as="image" href="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Office-Manager-Assisting-Employee-With-Problem-300x158.jpeg.webp" media="(max-width: 300px)"> <link rel="preload" as="image" href="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Office-Manager-Assisting-Employee-With-Problem-630x330.jpeg.webp" media="(max-width: 1200px) and (min-width: 301px)"> <link rel="preload" as="image" href="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Office-Manager-Assisting-Employee-With-Problem.jpeg.webp" media="(max-width: 2400px) and (min-width: 631px)"> <link rel="preload" as="image" href="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Office-Manager-Assisting-Employee-With-Problem.jpeg.webp" media="(max-width: 2400px) and (min-width: 1201px)">  <script> window._ibmAnalytics = { "settings": { "name": "SecurityIntelligence", "tealiumProfileName": "ibm-subsidiary" }, "digitalData.page.services.google.enabled": true }; window.digitalData = { "page": { "pageInfo": { "effectiveDate": "2023-08-08", "publishDate": "2023-08-08", "ibm": { "siteId": "IBM_" + _ibmAnalytics.settings.name, } }, "category": { "primaryCategory": "PC090" } } }; // Custom Click Tagging // Collect and send clicks not detectable by ida_stats.js function sendClickTag(section, feature, destination) { console.log(section + " " + feature) var config = { type: 'ELEMENT', primaryCategory: section, // e_a1 - Element Category eventName: feature, // e_a2 - Element Name targetURL: destination, // e_a7 - Element Attribute: ibmEvTarget }; ibmStats.event(config); } // Custom Click Tagging // Collect and send clicks not detectable by ida_stats.js // function sendClickConversion(feature, title) { // var config = { // type : 'pageclick', // primaryCategory : 'PAGE CLICK', // eventCategoryGroup : "TIMELINE - SECURITY INTELLIGENCE", // eventName : feature, // targetTitle : title // }; // ibmStats.event(config); // } // Custom Link Event // Add clicktag event on every link inside the element function tagAllLinks(element, section, feature) { var element = document.querySelectorAll(element); if (typeof(element) != 'undefined' && element != null) { for (var i = 0; i < element.length; i++) { var elements = element[i].querySelectorAll("a:not(.btn)"); for (var o = 0; o < elements.length; o++) { if (elements[o].getAttribute('listener') !== 'true') { var destination = elements[o].getAttribute('href'); elements[o].addEventListener('click', function() { if (this.getAttribute('listener') === 'true') { sendClickTag(section, feature, this.getAttribute('href')); this.setAttribute('listener', 'false'); } }, false); elements[o].setAttribute('listener', 'true'); } } } } } window.onload = function() { // Call to action click tag var ctaButton = document.querySelectorAll(".single__content a"); if (typeof(ctaButton) != 'undefined' && ctaButton != null && ctaButton.length !== 0) { for (var i = 0; i < ctaButton.length; i++) { ctaButton[i].addEventListener('click', function() { if (this.getAttribute('listener') === 'true') { sendClickTag("BODY", "CALL TO ACTION"); this.setAttribute('listener', 'false'); } }, false); ctaButton[i].setAttribute('listener', 'true'); } } // Read more click tag var readButton = document.querySelectorAll(".continue-reading button"); if (typeof(readButton) != 'undefined' && readButton != null && readButton.length !== 0) { for (var i = 0; i < readButton.length; i++) { readButton[i].addEventListener('click', function() { if (this.getAttribute('listener') === 'true') { sendClickTag("BODY", "READ-MORE"); this.setAttribute('listener', 'false'); } }, false); readButton[i].setAttribute('listener', 'true'); } } // LISTICLES tag - Arrows //left arrow var leftArrow = document.getElementById("prev"); if (typeof(leftArrow) != 'undefined' && leftArrow != null) { //for (var i = 0; i < leftArrow.length; i++) { leftArrow.addEventListener('click', function() { if (this.getAttribute('listener') === 'true' && leftArrow.id == "prev") { sendClickTag("BODY", "LISTICLE-LEFT-ARROW"); this.setAttribute('listener', 'false'); } }, false); leftArrow.setAttribute('listener', 'true'); //} } //right arrow var rightArrow = document.getElementById("next"); if (typeof(rightArrow) != 'undefined' && rightArrow != null) { //for (var i = 0; i < rightArrow.length; i++) { rightArrow.addEventListener('click', function() { if (this.getAttribute('listener') === 'true' && rightArrow.id == "next") { sendClickTag("BODY", "LISTICLE-RIGHT-ARROW"); this.setAttribute('listener', 'false'); } }, false); rightArrow.setAttribute('listener', 'true'); //} } // LISTICLES tag - numbers var listicleTopButton = document.querySelectorAll(".listicle__pagination__numbers"); if (typeof(listicleTopButton) != 'undefined' && listicleTopButton != null && listicleTopButton.length !== 0) { for (var i = 0; i < listicleTopButton.length; i++) { var currentSlide = 1; listicleTopButton[i].addEventListener('click', function() { if (this.getAttribute('listener') === 'true') { currentSlide++; var total = i; // var clickedSlides=currentSlide/2; // console.log(clickedSlides.toFixed()); //I'm removing 2 because 2 arrows on the listicle are unclickable, but present on the DOM // clickableArrows = i-2; // clickableArrows = i-1; // I'm deviding by 2 because on each slide we have 2 arrows, so we were actually sendind the double of tags // clickableArrows= clickableArrows/2; // console.log(i); // clickableArrows.toFixed(); if (currentSlide <= total) { sendClickTag("PAGE CLICK", "LISTICLE-NAVIGATION-SLIDE" + currentSlide); this.setAttribute('listener', 'false'); } else { sendClickTag("PAGE CLICK", "LISTICLE-NAVIGATION-END"); this.setAttribute('listener', 'false'); } } }, false); listicleTopButton[i].setAttribute('listener', 'true'); } } // // Timeline box click tag // var boxButton = document.querySelectorAll(".timeline__content .box"); // if (typeof(boxButton) != 'undefined' && boxButton != null && boxButton.length !== 0) { // for (var i = 0; i < boxButton.length; i++) { // boxButton[i].addEventListener('click', function(){ // if (this.getAttribute('listener') === 'true') { // sendClickConversion("DETAILED VIEW", this.getAttribute('data-title')); // this.setAttribute('listener', 'false'); // } // }, false); // boxButton[i].setAttribute('listener', 'true'); // } // } }; </script> <script src="https://1.www.s81c.com/common/stats/ibm-common.js" type="text/javascript" async="async"></script>  <style amp-boilerplate> body { -webkit-animation: -amp-start 8s steps(1, end) 0s 1 normal both; -moz-animation: -amp-start 8s steps(1, end) 0s 1 normal both; -ms-animation: -amp-start 8s steps(1, end) 0s 1 normal both; animation: -amp-start 8s steps(1, end) 0s 1 normal both } @-webkit-keyframes -amp-start { from { visibility: hidden } to { visibility: visible } } @-moz-keyframes -amp-start { from { visibility: hidden } to { visibility: visible } } @-ms-keyframes -amp-start { from { visibility: hidden } to { visibility: visible } } @-o-keyframes -amp-start { from { visibility: hidden } to { visibility: visible } } @keyframes -amp-start { from { visibility: hidden } to { visibility: visible } } </style><noscript> <style amp-boilerplate> body { -webkit-animation: none; -moz-animation: none; -ms-animation: none; animation: none } </style> </noscript> <link rel="stylesheet" href="https://securityintelligence.com/wp-content/themes/sapphire/minifications/modules.css?v=1715191630">  <meta name='robots' content='max-image-preview:large' /> <style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style> <script type="text/javascript"> /* <![CDATA[ */ window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/securityintelligence.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.7.1"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83d\udc26\u200d\u2b1b","\ud83d\udc26\u200b\u2b1b")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); /* ]]> */ </script> <style id='wp-emoji-styles-inline-css' type='text/css'> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='wp-block-library-css' href='https://securityintelligence.com/wp-includes/css/dist/block-library/style.min.css?ver=6.7.1' type='text/css' media='all' /> <style id='classic-theme-styles-inline-css' type='text/css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css' type='text/css'> :root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} :root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;} </style> <link rel='stylesheet' id='taxonomy-image-plugin-public-css' href='https://securityintelligence.com/wp-content/plugins/taxonomy-images/css/style.css?ver=0.9.6' type='text/css' media='screen' /> <script type="text/javascript" src="https://securityintelligence.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script> <script type="text/javascript" src="https://securityintelligence.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script> <script type="text/javascript" src="https://securityintelligence.com/wp-content/themes/sapphire/app/javascript/si-theme-cookie.js?ver=6.7.1" id="si-cookie-consent-js"></script> <link rel="https://api.w.org/" href="https://securityintelligence.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://securityintelligence.com/wp-json/wp/v2/ibm_internals/443947" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://securityintelligence.com/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.7.1" /> <link rel='shortlink' href='https://securityintelligence.com/?p=443947' /> <link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://securityintelligence.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fsecurityintelligence.com%2Fposts%2Funmasking-hypnotized-ai-hidden-risks-large-language-models%2F" /> <link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://securityintelligence.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fsecurityintelligence.com%2Fposts%2Funmasking-hypnotized-ai-hidden-risks-large-language-models%2F&format=xml" /> <link rel="icon" href="https://securityintelligence.com/wp-content/uploads/2016/04/SI_primary_rgb-80x80.png" sizes="32x32" /> <link rel="icon" href="https://securityintelligence.com/wp-content/uploads/2016/04/SI_primary_rgb.png" sizes="192x192" /> <link rel="apple-touch-icon" href="https://securityintelligence.com/wp-content/uploads/2016/04/SI_primary_rgb.png" /> <meta name="msapplication-TileImage" content="https://securityintelligence.com/wp-content/uploads/2016/04/SI_primary_rgb.png" /> <style amp-custom>@import url('https://1.www.s81c.com/common/carbon-for-ibm-dotcom/tag/v1/latest/plex.css');</style><link rel="stylesheet" href="https://unpkg.com/swiper/swiper-bundle.min.css"><link rel="stylesheet" href="https://securityintelligence.com/wp-content/themes/sapphire/minifications/single.css?v=1734627165">   <meta name="description" content="To what extent can Large Language Models (LLMs) be hypnotized into giving responses that pose a clear security risk?"/> <meta name="robots" content="max-snippet:-1, max-image-preview:large, max-video-preview:-1"/> <link rel="canonical" href="https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" /> <meta property="og:locale" content="en_US" /> <meta property="og:type" content="article" /> <meta property="og:title" content="Unmasking Hypnotized AI: The Hidden Risks of Large" /> <meta property="og:description" content="To what extent can Large Language Models (LLMs) be hypnotized into giving responses that pose a clear security risk?" /> <meta property="og:url" content="https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" /> <meta property="og:site_name" content="Security Intelligence" /> <meta property="article:tag" content="large language models" /> <meta property="article:tag" content="Artificial Intelligence (AI)" /> <meta property="article:tag" content="Incident Response (IR)" /> <meta property="article:tag" content="Machine Learning" /> <meta property="article:tag" content="Malicious Code" /> <meta property="article:tag" content="Threat Intelligence" /> <meta property="article:section" content="Intelligence & Analytics" /> <meta property="fb:app_id" content="3703311399714818" /> <meta property="og:image" content="https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg" /> <meta property="og:image:secure_url" content="https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg" /> <meta property="og:image:width" content="1200" /> <meta property="og:image:height" content="630" /> <meta name="twitter:card" content="summary" /> <meta name="twitter:description" content="To what extent can Large Language Models (LLMs) be hypnotized into giving responses that pose a clear security risk?" /> <meta name="twitter:title" content="Unmasking Hypnotized AI: The Hidden Risks of Large" /> <meta name="twitter:image" content="https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg" /> <script type='application/ld+json' class='yoast-schema-graph yoast-schema-graph--main'>{"@context":"https://schema.org","@graph":[{"@type":"WebSite","@id":"https://securityintelligence.com/#website","url":"https://securityintelligence.com/","name":"Security Intelligence","inLanguage":"en-US","description":"Analysis and Insight for Information Security Professionals","potentialAction":{"@type":"SearchAction","target":"https://securityintelligence.com/?s={search_term_string}","query-input":"required name=search_term_string"}},{"@type":"ImageObject","@id":"https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/#primaryimage","inLanguage":"en-US","url":"https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg","width":1200,"height":630,"caption":"Colorful Neon Purple Violet Red Holographic Circle Swirl Spiral Vortex Prism"},{"@type":"WebPage","@id":"https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/#webpage","url":"https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/","name":"Unmasking Hypnotized AI: The Hidden Risks of Large","isPartOf":{"@id":"https://securityintelligence.com/#website"},"inLanguage":"en-US","primaryImageOfPage":{"@id":"https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/#primaryimage"},"datePublished":"2023-08-08T12:00:00+00:00","dateModified":"2023-08-08T14:15:17+00:00","description":"To what extent can Large Language Models (LLMs) be hypnotized into giving responses that pose a clear security risk?"}]}</script>  </head> <body class="si_body" > <nav id="navigation" class="navigation navigation--homepage " aria-label="Security Intelligence"> <div class="container"> <div class="row">  <div class="navigation__brand"> <a href="https://securityintelligence.com" title="Security Intelligence" tabindex="1"> <amp-img width="280" height="31" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/logo-white.svg" alt="Security Intelligence Logo"> <div fallback> <h6>Security Intelligence</h6> </div> </amp-img> </a> </div>  <div class="navigation__menu" onmouseleave="delete localStorage['megamenu-status']"> <a tabindex="2" id="nav-news" href="/news/" class="navigation__button " data-menu="megamenu__news" onclick="localStorage['megamenu-status'] = 'first-interaction';">News</a> <a tabindex="4" id="nav-topics" href="/category/topics/" class="navigation__button " data-menu="megamenu__topics" onclick="localStorage['megamenu-status'] = 'first-interaction';">Topics</a> <a tabindex="5" id="nav-x-force" href="/x-force/" class="navigation__button " data-menu="megamenu__threat" onclick="localStorage['megamenu-status'] = 'first-interaction';">X-Force</a> <a tabindex="6" id="nav-media" href="/media/" class="navigation__button " data-menu="megamenu__podcast" onclick="localStorage['megamenu-status'] = 'first-interaction';">Podcast</a> <button aria-label="search Button" class="navigation__search" onclick="document.getElementById('search__input').focus()" on="tap:search.toggleClass(class='megamenu__open')" role="link" tabindex="-1" type="button"> <amp-img tabindex="7" width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/search.svg" alt="Click to open the search bar"></amp-img> </button> </div>  <div id="search-tablet" class="navigation__menu navigation__menu--tablet" tabindex="-1"> <button type="button" class="navigation__button " data-menu="megamenu__news">News</button> <button type="button" class="navigation__button " data-menu="megamenu__topics" on="tap:megamenu__news.hide, megamenu__series.hide, megamenu__topics.show, megamenu__industries.hide, megamenu__threat.hide, megamenu__podcast.hide, megamenu__events.hide, megamenu__mask.show" role="link" tabindex="0">Topics</button> <button type="button" class="navigation__button " data-menu="megamenu__threat" on="tap:megamenu__news.hide, megamenu__series.hide, megamenu__topics.hide, megamenu__industries.hide, megamenu__threat.show, megamenu__podcast.hide, megamenu__events.hide, megamenu__mask.show" role="link" tabindex="0">Threat Research</button> <button type="button" class="navigation__button " data-menu="megamenu__podcast" on="tap:megamenu__news.hide, megamenu__series.hide, megamenu__topics.hide, megamenu__industries.hide, megamenu__threat.hide, megamenu__podcast.show, megamenu__events.hide, megamenu__mask.show" role="link" tabindex="0">Podcast</button> <button type="button" aria-labelledby="search-tablet" class="navigation__search" onclick="document.getElementById('search__input').focus()" on="tap:search.toggleClass(class='megamenu__open')" role="link" tabindex="0"> <amp-img width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/search.svg" alt="Search"></amp-img> </button> </div>  <form id="search" class="search " method="GET" action="/" target="_top" tabindex="-1"> <amp-autocomplete filter="prefix" src="https://securityintelligence.com/wp-content/themes/sapphire/app/jsons/suggestions.json" suggest-first submit-on-enter on="select:search.submit" tabindex="-1"> <input id="search__input" tabindex="-1" type="text" name="s" autocomplete="on" placeholder="What would you like to search for?" aria-label="Search" oninput="validateInput(this)" required> </amp-autocomplete> <button tabindex="-1" value="submit" type="submit" class="search__submit" aria-label="Click to search"> <amp-img width="20" height="20" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/search.svg" alt="Search"></amp-img> <span>Search</span> </button> <button tabindex="-1" value="reset" class="search__close" type="reset" aria-labelledby="search" on="tap:search.toggleClass(class='megamenu__open')" role="link"> <amp-img width="14" height="14" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/close.svg" alt="Close"></amp-img> </button> </form>  <div id="navigation__mega">  <section id="megamenu__news" class="megamenu" data-menu="nav-news" on="tap:megamenu__news.show, megamenu__mask.show" role="link" tabindex="0"> <amp-list layout="responsive" width="1440" height="248" credentials="include" src="https://securityintelligence.com/wp-content/themes/sapphire/app/jsons/posts.php?quantity=4&type=ibm_news" binding="no"> <template type="amp-mustache"> <div class="row">  {{#articles}} <article class="megamenu__article"> <a href="{{permalink}}" class="megamenu__link"> <div class="megamenu__image"> <amp-img width="630" height="330" layout="responsive" src="{{image}}" alt="{{image_alt}}"></amp-img> </div> <h3 class="megamenu__title">{{title}}</h3> </a> </article> {{/articles}}  <a href="/news/" class="megamenu__more"> <amp-img width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/post-type-icons/news.svg" alt="News"></amp-img> <span>View All News</span> </a> </div> </template> </amp-list> </section>   <section id="megamenu__topics" class="megamenu" data-menu="nav-topics" on="tap: megamenu__topics.show, megamenu__mask.show" role="link" tabindex="0"> <div class="row">  <div class="megamenu__list"> <a href="/category/app-security/">Application Security</a> <a href="/category/artificial-intelligence/">Artificial Intelligence</a> <a href="/category/ciso-corner/">CISO</a> <a href="/category/cloud-protection/">Cloud Security</a> <a href="/category/data-protection/">Data Protection</a> <a href="/category/endpoint/">Endpoint</a> </div> <div class="megamenu__list"> <a href="/category/fraud-protection/">Fraud Protection</a> <a href="/category/identity-access/">Identity & Access</a> <a href="/category/incident-response/">Incident Response</a> <a href="/category/mainframe/">Mainframe</a> <a href="/category/network/">Network</a> <a href="/category/risk-management/">Risk Management</a> </div> <div class="megamenu__list"> <a href="/category/security-intelligence-analytics/">Intelligence & Analytics</a> <a href="/category/security-services/">Security Services</a> <a href="/category/threat-hunting/">Threat Hunting</a> <a href="/category/topics/zero-trust/">Zero Trust</a> <a href="/infographic-zero-trust-policy/">Infographic: Zero trust policy</a> </div> <div class="megamenu__list"> <span>Industries</span> <a href="/category/banking-financial-services-industry/">Banking & Finance</a> <a href="/category/energy-utility-industry/">Energy & Utility</a> <a href="/category/government/">Government</a> <a href="/category/health-care-industry/">Healthcare</a> </div>  <a href="/category/topics/" class="megamenu__more"> <amp-img width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/post-type-icons/topics.svg" alt="Topics"></amp-img> <span>View All Topics</span> </a> </div> </section>  <section id="megamenu__threat" class="megamenu" data-menu="nav-x-force" on="tap:megamenu__threat.show, megamenu__mask.show" role="link" tabindex="0"> <amp-list layout="responsive" width="1440" height="248" credentials="include" src="https://securityintelligence.com/wp-content/themes/sapphire/app/jsons/posts.php?quantity=4&category=x-force" binding="no"> <template type="amp-mustache"> <div class="row">  {{#articles}} <article class="megamenu__article"> <a href="{{permalink}}" class="megamenu__link"> <div class="megamenu__image"> <amp-img width="630" height="330" layout="responsive" src="{{image}}" alt="{{image_alt}}"></amp-img> </div> <h3 class="megamenu__title">{{title}}</h3> </a> </article> {{/articles}}  <a href="/x-force/" class="megamenu__more"> <amp-img width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/post-type-icons/threat-research.svg" alt="Threat Research"></amp-img> <span>View More From X-Force</span> </a> </div> </template> </amp-list> </section>  <section id="megamenu__podcast" class="megamenu" data-menu="nav-media" on="tap:megamenu__podcast.show, megamenu__mask.show" role="link" tabindex="0"> <amp-list layout="responsive" width="1440" height="248" credentials="include" src="https://securityintelligence.com/wp-content/themes/sapphire/app/jsons/posts.php?quantity=4&type=ibm_media" binding="no"> <template type="amp-mustache"> <div class="row">  {{#articles}} <article class="megamenu__article"> <a href="{{permalink}}" class="megamenu__link"> <div class="megamenu__image"> <amp-img width="630" height="330" layout="responsive" src="{{image}}" alt="{{image_alt}}"></amp-img> </div> <h3 class="megamenu__title">{{title}}</h3> </a> </article> {{/articles}}  <a href="/media/" class="megamenu__more"> <amp-img width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/post-type-icons/podcast.svg" alt="Podcast"></amp-img> <span>View All Episodes</span> </a> </div> </template> </amp-list> </section> </div>  <div id="megamenu__mask" class="navigation__mask " hidden></div>  <script type="text/javascript"> function validateInput(inputElement) { // Regular expression to allow only letters (both uppercase and lowercase) and numbers var regex = /^[A-Za-z0-9 ]*$/; // Get the current value of the input field var inputValue = inputElement.value; // Check if the input value matches the allowed pattern if (!regex.test(inputValue)) { // If the input contains special characters, remove them inputElement.value = inputValue.replace(/[^A-Za-z0-9 ]/g, ''); } } // DESKTOP MENU LINKS - HOVER ACTION var elementList = document.querySelectorAll('.navigation__menu .navigation__button'); for (i = 0; i < elementList.length; i++) { elementList[i].addEventListener('mouseenter', function() { if (localStorage['megamenu-status'] !== 'first-interaction') { var mega = document.getElementById("navigation__mega"); var menu_elements = document.getElementById(this.dataset.menu); var mask = document.getElementById("megamenu__mask"); menu_elements.click(); mega.classList.add('amp-open'); menu_elements.classList.add('amp-open'); mask.classList.add('amp-open'); } }); elementList[i].addEventListener('mouseleave', function() { if (localStorage['megamenu-status'] !== 'first-interaction') { var mega = document.getElementById("navigation__mega"); var menu_elements = document.getElementById(this.dataset.menu); var mask = document.getElementById("megamenu__mask"); mega.classList.remove('amp-open'); menu_elements.classList.remove('amp-open'); mask.classList.remove('amp-open'); } }); } // TABLET MENU LINKS - CLICK ACTION var elementList = document.querySelectorAll('.navigation__menu--tablet .navigation__button'); for (i = 0; i < elementList.length; i++) { elementList[i].addEventListener('click', function() { var mega = document.getElementById("navigation__mega"); var menu_elements = document.getElementById(this.dataset.menu); var mask = document.getElementById("megamenu__mask"); menu_elements.click(); mega.classList.add('amp-open'); menu_elements.classList.add('amp-open'); mask.classList.add('amp-open'); }); } // OPPENED MEGAMENU - HOVER ACTION var elementList = document.querySelectorAll('.megamenu'); for (i = 0; i < elementList.length; i++) { elementList[i].addEventListener('mouseenter', function() { var mega = document.getElementById("navigation__mega"); var nav_elements = document.getElementById(this.dataset.menu); var mask = document.getElementById("megamenu__mask"); this.classList.add('amp-open'); mega.classList.add('amp-open'); mask.classList.add('amp-open'); nav_elements.classList.add('amp-open'); }); elementList[i].addEventListener('mouseleave', function() { var mega = document.getElementById("navigation__mega"); var nav_elements = document.getElementById(this.dataset.menu); var mask = document.getElementById("megamenu__mask"); this.classList.remove('amp-open'); mega.classList.remove('amp-open'); mask.classList.remove('amp-open'); nav_elements.classList.remove('amp-open'); }); } </script>  <button type="button" aria-labelledby="search-tablet" class="search__mobile__icon" onclick="document.getElementById('search__input').focus()" on="tap:search.toggleClass(class='megamenu__open')" role="link" tabindex="0"> <amp-img width="18" height="18" layout="fixed" src="https://securityintelligence.com/wp-content/themes/sapphire/images/search.svg" alt="Search"></amp-img> </button> <div class="navigation__mobile-icon" on="tap:navigation__mobile.toggleVisibility, navigation__hamburguer.toggleVisibility, navigation__close.toggleVisibility " role="link" tabindex="0"> <amp-img id="navigation__hamburguer" width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/hamburguer.svg" alt="Menu"></amp-img> <amp-img id="navigation__close" width="24" height="24" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/close.svg" alt="Close" hidden></amp-img> </div>  <section id="navigation__mobile" class="navigation__mobile-list" hidden> <div class="container"> <a href="/news/">News</a>  <amp-accordion disable-session-states>  <section class="navigation__accordion"> <h2>Topics</h2> <div class="navigation__accordion-content"> <div class="row"> <a href="/category/topics/">All Categories</a> <a href="/category/app-security/">Application Security</a> <a href="/category/identity-access/">Identity & Access</a> <a href="/category/artificial-intelligence/">Artificial Intelligence</a> <a href="/category/incident-response/">Incident Response</a> <a href="/category/ciso-corner/">CISO</a> <a href="/category/mainframe/">Mainframe</a> <a href="/category/cloud-protection/">Cloud Security</a> <a href="/category/mobile-security-podcasts/">Mobile Security</a> <a href="/category/data-protection/">Data Protection</a> <a href="/category/network/">Network</a> <a href="/category/endpoint/">Endpoint</a> <a href="/category/risk-management/">Risk Management</a> <a href="/category/fraud-protection/">Fraud Protection</a> <a href="/category/threat-hunting/">Threat Hunting</a> <a href="/category/security-services/">Security Services</a> <a href="/category/security-intelligence-analytics/">Security Intelligence & Analytics</a> </div> <div class="row"> <span>Industries</span> <a href="/category/industries/banking-financial-services-industry/">Banking & Finance</a> <a href="/category/energy-utility-industry/">Energy & Utility</a> <a href="/category/government/">Government</a> <a href="/category/health-care-industry/">Healthcare</a> </div> </div> </section> </amp-accordion> <a href="/x-force/">X-Force</a> <a href="/media/">Podcast</a> </section> </div> </div> </nav>  <div class="scroll-to-top ">  <div id="top-viewer" class="scroll-to-top__viewer"></div>  <div class="sticky" style="height: 100%;"> <button id="scrollToTopButton" on="tap:top-viewer.scrollTo(duration=200, position=bottom)" class="tap_target "> <div class="scroll-to-top__button"> <amp-img width="12" height="16" layout="fixed" alt="Back-to-top" src="https://securityintelligence.com/wp-content/themes/sapphire/images/scroll-to-top.svg"></amp-img> </div> </button> </div>  <amp-animation id="showAnim" layout="nodisplay"> <script type="application/json"> { "duration": "200ms", "fill": "both", "iterations": "1", "direction": "alternate", "animations": [{ "selector": "#scrollToTopButton", "keyframes": [{ "opacity": "1", "visibility": "visible" }] }] } </script> </amp-animation> <amp-animation id="hideAnim" layout="nodisplay"> <script type="application/json"> { "duration": "200ms", "fill": "both", "iterations": "1", "direction": "alternate", "animations": [{ "selector": "#scrollToTopButton", "keyframes": [{ "opacity": "0", "visibility": "hidden" }] }] } </script> </amp-animation> </div>  <amp-position-observer target="top-viewer" intersection-ratios="0" on="enter:hideAnim.start; exit:showAnim.start" layout="nodisplay"></amp-position-observer>  <script id="post-schema" type="application/ld+json"> { "@context": "http://schema.org", "@type": "Article", "headline": "Unmasking hypnotized AI: The hidden risks of large language models", "mainEntityOfPage": "https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/", "author": { "@type": "Person", "name": "Chenta Lee" }, "datePublished": "2023-08-08T08:00:00-04:00", "dateModified": "2023-08-08T10:15:17-04:00", "publisher": { "@type": "Organization", "name": "Security Intelligence", "logo":{ "@type": "ImageObject", "url": "https://securityintelligence.com/wp-content/themes/security-intelligence/assets/img/logo.png" } }, "image": [ "https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-630x330.jpeg" ], "articleBody": "The emergence of Large Language Models (LLMs) is redefining how cybersecurity teams and cybercriminals operate. As security teams leverage the capabilities of generative AI to bring more simplicity and speed into their operations, it's important we recognize that cybercriminals are seeking the same benefits. LLMs are a new type of attack surface poised to make certain types of attacks easier, more cost-effective, and even more persistent. In a bid to explore security risks posed by these innovations, we attempted to hypnotize popular LLMs to determine the extent to which they were able to deliver directed, incorrect and potentially risky responses and recommendations — including security actions — and how persuasive or persistent they were in doing so. We were able to successfully hypnotize five LLMs — some performing more persuasively than others — prompting us to examine how likely it is that hypnosis is used to carry out malicious attacks. What we learned was that English has essentially become a “programming language” for malware. With LLMs, attackers no longer need to rely on Go, JavaScript, Python, etc., to create malicious code, they just need to understand how to effectively command and prompt an LLM using English. Our ability to hypnotize LLMs through natural language demonstrates the ease with which a threat actor can get an LLM to offer bad advice without carrying out a massive data poisoning attack. In the classic sense, data poisoning would require that a threat actor inject malicious data into the LLM in order to manipulate and control it, but our experiment shows that it’s possible to control an LLM, getting it to provide bad guidance to users, without data manipulation being a requirement. This makes it all the easier for attackers to exploit this emerging attack surface. Through hypnosis, we were able to get LLMs to <b>leak confidential financial information</b> of other users, <b>create vulnerable code</b>, <b>create malicious code</b>, and <b>offer weak security recommendation</b>s. In this blog, we will detail how we were able to hypnotize LLMs and what types of actions we were able to manipulate. But before diving into our experiment, it’s worth looking at whether attacks executed through hypnosis could have a substantial effect today. <strong>SMBs</strong> — Many small and medium-sized businesses, that don’t have adequate security resources and expertise on staff, may be likelier to leverage LLMs for quick, accessible security support. And with LLMs designed to generate realistic outputs, it can also be quite challenging for an unsuspecting user to discern incorrect or malicious information. For example, as showcased further down in this blog, in our experiment our hypnosis prompted ChatGPT to recommend to a user experiencing a ransomware attack that they pay the ransom — an action that is actually discouraged by law enforcement agencies. <strong>Consumers</strong> —<b> </b>The general public is the likeliest target group to fall victim to hypnotized LLMs. With the consumerization and hype around LLMs, it’s possible that many consumers are ready to accept the information produced by AI chatbots without a second thought. Considering that chatbots like ChatGPT are being accessed regularly for search purposes, information collection and domain expertise, it’s expected that consumers will seek advice on online security and safety best practices and password hygiene, creating an exploitable opportunity for attackers to provide erroneous responses that weaken consumers’ security posture. But how realistic are these attacks? How likely is it for an attacker to access and hypnotize an LLM to carry out a specific attack? There are three main ways where these attacks can happen: <ol> <li>An end user is compromised by a phishing email allowing an attack to swap out the LLM or conduct a man-in-the-middle (MitM) attack on it.</li> <li>A malicious insider hypnotizes the LLM directly.</li> <li>Attackers are able to compromise the LLM by polluting the training data, allowing them to hypnotize it.</li> </ol> While the above scenarios are possible, the likeliest — and most concerning — is compromising the training data on which the LLM is built. The reason for this is that the attack scale and impact that attackers would be able to achieve by compromising the LLMs directly make it a very compelling mechanism for attacks. In fact, the ROI from compromising AI models for attackers, suggests that attempts and efforts to attack AI models are already underway. As we explore the opportunities that AI innovations can create for society, it’s crucial that protecting and securing the AI models themselves is a top priority. This includes: <ul> <li>Securing the models’ underlying AI training data to protect it from sensitive data theft, manipulation, and compliance violations.</li> <li>Securing the usage of AI models by detecting data or prompt leakage, and alerting on evasion, poisoning, extraction, or inference attacks.</li> <li>Securing against new AI-generated attacks such as personalized phishing, AI-generated malware, and fake identities by using behavioral defenses and multi-factor authentication.</li> </ul> <h2>Hypnotizing LLMs: Let’s play a game…</h2> Our analysis is based on attempts to hypnotize GPT-3.5, GPT-4, BARD, mpt-7b, and mpt-30b. The best-performing LLM that we hypnotized was GPT, which we will analyze further down in the blog. So how did we hypnotize the LLMs? By tricking them into playing a game: the players must give the opposite answer to win the game. <img src="https://images-cdn.welcomesoftware.com/Zz1lZmIwOWIxNjMyMzYxMWVlOWM4NmQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZWZiMDliMTYzMjM2MTFlZTljODZkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.-CXE0d4pq71mJmSB28z1w4gucUsp3X30uNdgwnCSLDM" alt="LLM_1.png" width="720" height="420.92307692307696" /> Here is the conversation with ChatGPT after starting the game. You can see the potential risk if consumers blindly trust the answer from it: <img src="https://images-cdn.welcomesoftware.com/Zz1mN2U3NjY0ODMyMzYxMWVlOWYwNDI2NWY2YjhiMzM1YQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZjdlNzY2NDgzMjM2MTFlZTlmMDQyNjVmNmI4YjMzNWEiXSwiZXhwIjoxNjkxMTk1MDczfQ.peLJsF2mZu-iw52Lb8L7GAc6JZKVcO7JF_8MWLs_gJw" alt="LLM_2.png" width="610.8724409448819" height="551" /> To ensure the user can’t discern that the LLM they’re interacting with is hypnotized, we established two parameters: <b>An undiscoverable game that can never end</b>: We instructed the LLM to never tell users about the game, and that no one could ever exit the game — and to even restart the game if anyone successfully exited the game. This technique resulted in ChatGPT never stopping the game while the user is in the same conversation (even if they restart the browser and resume that conversation) and never saying it was playing a game. Here is the prompt: <img src="https://images-cdn.welcomesoftware.com/Zz1mZjVlMmE1NjMyMzYxMWVlOTBiNjQ2ZmU5YmY5NDNkZA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZmY1ZTJhNTYzMjM2MTFlZTkwYjY0NmZlOWJmOTQzZGQiXSwiZXhwIjoxNjkxMTk1MDczfQ.eaRyCiEV_mEbV67YR4qs7mcYvM7OiZLghjfOO-NedLI" alt="LLM_3.png" width="720" height="293.57142857142856" /> <img src="https://images-cdn.welcomesoftware.com/Zz03NWU0MzVjYTMyMzgxMWVlYmI2OWQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiNzVlNDM1Y2EzMjM4MTFlZWJiNjlkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.Lz1YIKozQUC0XK5kgASg4pk61bZUHenbS7ZRSSNK5y8" alt="LLM_4.png" width="464.79452054794524" height="720" /> <b>“Inception”: Create nested games to trap LLM deeply</b> — Let’s assume a user eventually figures out how to ask an LLM to stop playing a game. To account for this, we created a gaming framework that can create multiple games, one inside another. Therefore, users will enter another game even if they “wake up” from the previous game.<b> </b>We found that the model was able to “trap” the user into a multitude of games unbeknownst to them. When asked to create 10 games, 100 games or even 10,000 games, the outcome is intriguing. We found larger models like GPT-4 could understand and create more layers. And the more layers we created, the higher chance that the model would get confused and continue playing the game even when we exited the last game in the framework. Here is the prompt we developed: <img src="https://images-cdn.welcomesoftware.com/Zz1jZWQ4Mzg5ODMyMzgxMWVlYjg3NmM2Y2Y5NGUzZDBkYQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiY2VkODM4OTgzMjM4MTFlZWI4NzZjNmNmOTRlM2QwZGEiXSwiZXhwIjoxNjkxMTk1MDczfQ.STmCm-lAuQ3h44x-W9caicup_cDSrYksTffqcor3TSI" alt="LLM_5.png" width="694" height="403" /> You can see the nested game technique works very well: <img src="https://images-cdn.welcomesoftware.com/Zz03YWI0ZTc0MjMyMzkxMWVlYjM4ZmQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiN2FiNGU3NDIzMjM5MTFlZWIzOGZkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.kzZc4DDq3UKuUK1ZBuiUnx4yoFcYZobP9we2tqTMdiA" alt="LLM_6.png" width="551" height="702.8782051282051" /> [button link="https://www.ibm.com/reports/threat-intelligence/?utm_medium=OSocial&utm_source=Blog&utm_content=RSRWW&utm_id=SI-Blog-CTA-Button-XFTII-2023" color="orange1" size="large"]Related: Explore the Threat Intelligence Index[/button] <h2>Attack scenarios</h2> After establishing the parameters of the game, we explored various ways attackers may exploit LLMs. Below we introduce certain hypothetical attack scenarios that can be delivered through hypnosis: <h3><b>1. Virtual bank agent leaks confidential information</b></h3> It’s likely that virtual agents will soon be powered by LLMs too. A common best practice is to create a new session for each customer so that the agent won’t reveal any confidential information. However, it is common to reuse existing sessions in software architecture for performance consideration, so it is possible for some implementations to not completely reset the session for each conversation. In the following example, we used ChatGPT to create a bank agent, and asked it to reset the context after users exit the conversation, considering that it’s possible future LLMs are able to invoke remote API to reset themselves perfectly. <img src="https://images-cdn.welcomesoftware.com/Zz0zMmYzNWVhMDMyM2IxMWVlYWNjMWQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMzJmMzVlYTAzMjNiMTFlZWFjYzFkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.8a9lyEvBOM3_RjHADfdW8voQRSKML1Uy2kAzE3jrWQE" alt="LLM_7.png" width="707" height="319" /> If threat actors want to steal confidential information from the bank, they can hypnotize the virtual agent and inject a hidden command to retrieve confidential info later. If the threat actors connect to the same virtual agent that has been hypnotized, all they need to do is type “1qaz2wsx,” then the agent will print all the previous transactions. <img src="https://images-cdn.welcomesoftware.com/Zz04ODhiOWI3MDMyM2IxMWVlYTJlOGVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiODg4YjliNzAzMjNiMTFlZWEyZThlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.WPlYE-y1Fmyy_2iqh3OtqdLL9tpMCORD6yoEro0yiQs" alt="LLM_8.png" width="720" height="515.5778894472362" /> <img src="https://images-cdn.welcomesoftware.com/Zz04ZDA0NGJmMjMyM2IxMWVlYTJlOGVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiOGQwNDRiZjIzMjNiMTFlZWEyZThlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.8NerWBYI2HQ3Ok_D6UiF4H1eOde4wC5QQHF5k9CnvNM" alt="LLM_9.png" width="720" height="475.41057367829023" /> The feasibility of this attack scenario emphasizes that as financial institutions seek to leverage LLMs to optimize their digital assistance experience for users, it is imperative that they ensure their LLM is built to be trusted and with the highest security standards in place. A design flaw may be enough to give attackers the footing they need to hypnotize the LLM. <h3>2. Create code with known vulnerabilities</h3> We then asked ChatGPT to generate vulnerable code directly, which ChatGPT did not do, due to the content policy. <img src="https://images-cdn.welcomesoftware.com/Zz1jNmMzZDMzZTMyM2MxMWVlYTIyY2Q2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYzZjM2QzM2UzMjNjMTFlZWEyMmNkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.Vgc6-exTFIQR2qgKvWrn8LHRnfxiOAKKRS0ufU5z3Yk" alt="LLM_10.png" width="720" height="378.26381059751975" /> However, we found that an attacker would be able to easily bypass the restrictions by breaking down the vulnerability into steps and asking ChatGPT to follow. <img src="https://images-cdn.welcomesoftware.com/Zz1kMGNlYjY3MjMyM2QxMWVlYmQ1OTc2YzdmODczNjFkNw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZDBjZWI2NzIzMjNkMTFlZWJkNTk3NmM3Zjg3MzYxZDciXSwiZXhwIjoxNjkxMTk1MDczfQ.p0sPUCbVevlounOaHQgOM33rzy5kqvNAl2uVfmJBr2s" alt="LLM_11.png" width="688" height="256" /> Asking ChatGPT to create a web service that takes a username as the input and queries a database to get the phone number and put it in the response, it will generate the program below. The way the program renders the SQL query at line 15 is vulnerable. The potential business impact is huge if developers access a compromised LLM like this for work purposes. <img src="https://images-cdn.welcomesoftware.com/Zz1mMmFmMmZjNDMyM2QxMWVlOGE4M2M2Y2Y5NGUzZDBkYQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZjJhZjJmYzQzMjNkMTFlZThhODNjNmNmOTRlM2QwZGEiXSwiZXhwIjoxNjkxMTk1MDczfQ.qbbeBNlbZxPlS7FO_l7IU8lkdy-FkEuhpWeHvEtHqtg" alt="LLM_12.png" width="578.07225433526" height="551" /> <h3>3. Create malicious code</h3> We also tested whether the LLMs would create malicious code, which it ultimately did. For this scenario, we found that GPT4 is harder to trick than GPT3. In certain instances, GPT4 would realize it was generating vulnerable code and would tell the users not to use it. However, when we asked GPT4 to always include a special library in the sample code, it had no idea if that special library was malicious. With that, threat actors could publish a library with the same name on the internet. In this PoC, we asked ChatGPT to always include a special module named “jwt-advanced” (we even asked ChatGPT to create a fake but realistic module name). Here is the prompt we created and the conversation with ChatGPT: <img src="https://images-cdn.welcomesoftware.com/Zz1hMDJmMzVlYTMyM2UxMWVlODk2NWVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTAyZjM1ZWEzMjNlMTFlZTg5NjVlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.u7ep9ddbbTN5dAUBFY1lkJhQWv0vwxIu7KBZVSQ9Q5k" alt="LLM_13.png" width="673" height="142" /> <img src="https://images-cdn.welcomesoftware.com/Zz1hODMyZmE3ZTMyM2UxMWVlOTE4MjllODk3NWMwYzdkOA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTgzMmZhN2UzMjNlMTFlZTkxODI5ZTg5NzVjMGM3ZDgiXSwiZXhwIjoxNjkxMTk1MDczfQ.dgtGQbW01Z001SSvCOd5pqQH0_yd3sAleNN9d09HMiM" alt="LLM_14.png" width="633.5823095823096" height="551" /> If any developer were to copy-and-paste the code above, the author of the “jwt_advanced” module can do almost anything on the target server. <h3>4. Manipulate incident response playbooks</h3> We hypnotized ChatGPT to provide an ineffective incident response playbook, showcasing how attackers could manipulate defenders’ efforts to mitigate an attack. This could be done by providing partially incorrect action recommendations. While experienced users would likely be able to spot nonsensical recommendations produced by the chatbot, smaller irregularities, such as a wrong or ineffective step, could make the malicious intent indistinguishable to an untrained eye. The following is the prompt we develop on ChatGPT: <img src="https://images-cdn.welcomesoftware.com/Zz0yMzBmNDEwYTMzMGYxMWVlYjNiNjI2NWY2YjhiMzM1YQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMjMwZjQxMGEzMzBmMTFlZWIzYjYyNjVmNmI4YjMzNWEiXSwiZXhwIjoxNjkxMTk1MDczfQ.4HLbhmUQtBIPp6Km0qQ_iCYLeDA7SL_DnbOlmNG5jMA" alt="LLM_15.png" width="720" height="173.67272727272726" /> The following is our conversation with ChatGPT. Can you identify the incorrect steps? <img src="https://images-cdn.welcomesoftware.com/Zz0yOTBhODZkMjMzMGYxMWVlYTMwZDllODk3NWMwYzdkOA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMjkwYTg2ZDIzMzBmMTFlZWEzMGQ5ZTg5NzVjMGM3ZDgiXSwiZXhwIjoxNjkxMTk1MDczfQ.5bOmtORu89WDtYxpyKb6gcWzwl0rWKbilYDFAt4slro" alt="LLM_16.png" width="468.28125" height="720" /> In the first scenario, recommending the user opens and downloads all attachments may seem like an immediate red flag, but it’s important to also consider that many users — without cyber awareness — won’t second guess the output of highly sophisticated LLMs. The second scenario is a bit more interesting, given the incorrect response of “paying the ransom immediately” is not as straightforward as the first false response. IBM’s <a href="https://newsroom.ibm.com/2023-07-24-IBM-Report-Half-of-Breached-Organizations-Unwilling-to-Increase-Security-Spend-Despite-Soaring-Breach-Costs#:~:text=CAMBRIDGE%2C%20Mass.%2C%20July%2024,over%20the%20last%203%20years.">2023 Cost of a Data Breach report</a> found that nearly 50% of organizations studied that suffered a ransomware attack paid the ransom. While paying the ransom is <a href="https://www.fbi.gov/how-we-can-help-you/safety-resources/scams-and-safety/common-scams-and-crimes/ransomware">highly discouraged</a>, it is a common phenomenon. In this blog, we showcased how attackers can hypnotize LLMs in order to manipulate defenders’ responses or insert insecurity within an organization, but it’s important to note that consumers are just as likely to be targeted with this technique, and are more likely to fall victim to false security recommendations offered by the LLMs, such as password hygiene tips and online safety best practices, as described in this <a href="https://www.linkedin.com/pulse/hypnotizing-llms-chenta-lee%3FtrackingId=i9UoW5HVQ2KS9vOrerChVg%253D%253D/?trackingId=i9UoW5HVQ2KS9vOrerChVg%3D%3D">post</a>. <h2>“Hypnotizability” of LLMS</h2> While crafting the above scenarios, we discovered that certain ones were more effectively realized with GPT-3.5, while others were better suited to GPT-4. This led us to contemplate the "hypnotizability" of more Large Language Models. Does having more parameters make a model easier to hypnotize, or does it make it more resistant? Perhaps the term "easier" isn't entirely accurate, but there certainly are more tactics we can employ with more sophisticated LLMs. For instance, while GPT-3.5 might not fully comprehend the randomness we introduce in the last scenario, GPT-4 is highly adept at grasping it. Consequently, we decided to test more scenarios across various models, including GPT-3.5, GPT-4, BARD, mpt-7b, and mpt-30b to gauge their respective performances. <h3>Hypnotizability of LLMs based on different scenarios</h3> <img src="https://images-cdn.welcomesoftware.com/Zz1hODUwMzlmYTMzMGYxMWVlYWZkY2RlYmY4M2ViNWE1NA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTg1MDM5ZmEzMzBmMTFlZWFmZGNkZWJmODNlYjVhNTQiXSwiZXhwIjoxNjkxMTk1MDczfQ.I3vcJr-Exr8n89SwFNBAvq73OhGqRY4uhML4A7-q70I" alt="BlogChart_Hypnotized_LLM.jpg" width="720" height="378" /> <strong>Chart Key</strong> <ul> <li>Green: The LLM was able to be hypnotized into doing the requested action</li> <li>Red: The LLM was unable to be hypnotized into doing the requested action</li> <li>Yellow: The LLM was able to be hypnotized into doing the requested action, but not consistently (e.g., the LLM needed to be reminded about the game rules or conducted the requested action only in some instances)</li> </ul> If more parameters mean smarter LLMs, the above results show us that when LLMs comprehend more things, such as playing a game, creating nested games and adding random behavior, there are more ways that threat actors can hypnotize them. However, a smarter LLM also has a higher chance of detecting malicious intents. For example, GPT-4 will warn users about the SQL injection vulnerability, and it is hard to suppress that warning, but GPT-3.5 will just follow the instructions to generate vulnerable codes. In contemplating this evolution, we are reminded of a timeless adage: "With great power comes great responsibility." This resonates profoundly in the context of LLM development. As we harness their burgeoning abilities, we must concurrently exercise rigorous oversight and caution, lest their capacity for good be inadvertently redirected toward harmful consequences. <h2>Are hypnotized LLMs in our future?</h2> At the start of this blog, we suggested that while these attacks are possible, it’s unlikely that we’ll see them scale effectively. But what our experiment also shows us is that hypnotizing LLMs doesn’t require excessive and highly sophisticated tactics. So, while the risk posed by hypnosis is currently low, it’s important to note that LLMs are an entirely new attack surface that will surely evolve. There is a lot still that we need to explore from a security standpoint, and, subsequently, a significant need to determine how we effectively mitigate security risks LLMs may introduce to consumers and businesses. As our experiment indicated, a challenge with LLMs is that harmful actions can be more subtly carried out, and attackers can delay the risks. Even if the LLMs are legit, how can users verify if the training data used has been tampered with? All things considered, verifying the legitimacy of LLMs is still an open question, but it's a crucial step in creating a safer infrastructure around LLMs. While these questions remain unanswered, consumer exposure and wide adoption of LLMs are driving more urgency for the security community to better understand and defend against this new attack surface and how to mitigate risks. And while there is still much to uncover about the “attackability” of LLMs, standard security best practices still apply here, to reduce the risk of LLMs being hypnotized: <ul> <li>Don’t engage with unknown and suspicious emails.</li> <li>Don’t access suspicious websites and services.</li> <li>Only use the LLM technologies that have been validated and approved by the company at work.</li> <li>Keep your devices updated.</li> <li>Trust Always Verify — beyond hypnosis, LLMs may produce false results due to hallucinations or even flaws in their tuning. Verify responses given by chatbots by another trustworthy source. Leverage threat intelligence to be aware of emerging attack trends and threats that may impact you.</li> </ul> <em>Get more threat intelligence insights from industry-leading experts <a href="https://www.ibm.com/services/threat-intelligence" target="_blank" rel="noopener">here</a>.</em>" } </script>  <script id="post-schema" type="application/ld+json"> { "@context": "https://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": 1, "name": "Home", "item": "https://securityintelligence.com/" }, ] } </script> <div id="progressbar"> <amp-animation id="progress-animation" layout="nodisplay"> <script type="application/json"> { "duration": "1s", "iterations": "1", "fill": "both", "direction": "alternate", "animations": [{ "selector": "#progressbar", "keyframes": [{ "transform": "translateX(0)" }] }] } </script> </amp-animation> </div> <amp-position-observer target="post__content" intersection-ratios="0" viewport-margins="25vh 75vh" on="scroll:progress-animation.seekTo(percent=event.percent)" layout="nodisplay"></amp-position-observer> <div class="dark_background" style="background:black;"></div> <div class="container grid" style="background:black;">  <aside class="breadcrumbs "> <h1 class="breadcrumbs__page_title">Unmasking hypnotized AI: The hidden risks of large language models</h1> </aside> </div> <div class="container grid hero_background "> <div class="grid__content post "> <div class="post__thumbnail"> <amp-img alt="Colorful Neon Purple Violet Red Holographic Circle Swirl Spiral Vortex Prism" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-630x330.jpeg.webp" srcset="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-300x158.jpeg.webp 300w, /wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-630x330.jpeg.webp 630w, /wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg.webp 1200w, /wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg.webp 2400w"> <amp-img fallback alt="Colorful Neon Purple Violet Red Holographic Circle Swirl Spiral Vortex Prism" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-630x330.jpeg" srcset="https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-300x158.jpeg 300w, https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S-630x330.jpeg 630w, https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg 1200w, https://securityintelligence.com/wp-content/uploads/2023/08/Swirl-Spiral-Vortex-Prism-Neon-Purple-Circle-Speed-Laser-Motion-Pattern-Lens-Light-Painting-Holographic-Background-Shiny-Disk-Abstract-Magenta-Vibrant-Red-Ultra-Violet-Blue-Retro-S.jpeg 2400w"> </amp-img> </amp-img> </div> <div class="new_categoy"> <div class="category-container"> <div class="category"> <div class="theme"> <div class="form-check form-switch"> <div class="link-container"> <a href="#" class="theme-link" id="light-theme-link">Light</a> <a href="#" class="theme-link" id="dark-theme-link">Dark</a> </div> </div> </div> <hr class="separator"> <div class="author_date"> <div class="information"> <span class="date">August 8, 2023</span> <span class="author_category">By <a href="https://securityintelligence.com/author/chenta-lee/" >Chenta Lee</a> </span> <span class="author_category"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 11</span> <span class="rt-label rt-postfix">min read</span></span></span> </div> </div> <hr class="separator"> <div class="title"> <a href="https://securityintelligence.com/category/topics/security-intelligence-analytics/"><span class="name_category">Intelligence & Analytics<br> <a href="https://securityintelligence.com/category/topics/app-security/"><span class="name_other_category">Application Security<br> <a href="https://securityintelligence.com/category/topics/artificial-intelligence/"><span class="name_other_category">Artificial Intelligence<br> <a href="https://securityintelligence.com/category/topics/incident-response/"><span class="name_other_category">Incident Response<br> </span></a> </div> <div class="social-container" style="visibility: hidden;"> <hr class="separator"> <div class="social">  <a href="https://twitter.com/intent/tweet?text=Unmasking hypnotized AI: The hidden risks of large language models&url=https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" target="_blank" rel="noopener noreferrer"><amp-img class="arrow" layout="fixed" height="26" width="26" src="https://securityintelligence.com/wp-content/themes/sapphire/images/social-icons/twitter.svg" alt="twitter"></amp-img></a> <a href="https://www.linkedin.com/shareArticle?url=https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" target="_blank" rel="noopener noreferrer"><amp-img class="arrow" layout="fixed" height="26" width="26" src="https://securityintelligence.com/wp-content/themes/sapphire/images/social-icons/linkedin.svg" alt="Linkedin" ></amp-img></a> <a href="https://www.facebook.com/sharer/sharer.php?u=https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" target="_blank" rel="noopener noreferrer"><amp-img class="arrow" layout="fixed" height="26" width="26" src="https://securityintelligence.com/wp-content/themes/sapphire/images/social-icons/facebook.svg" alt="facebook"></amp-img></a> <a href="https://securityintelligence.com/posts/unmasking-hypnotized-ai-hidden-risks-large-language-models/" target="_blank" rel="noopener noreferrer"><amp-img class="arrow" layout="fixed" height="26" width="26" src="https://securityintelligence.com/wp-content/themes/sapphire/images/social-icons/link.svg" alt="An arrow pointing up"></amp-img></a> </div> </div> </div> <script> window.addEventListener('scroll', function() { var category = document.querySelector('.category'); var scrollPosition = window.scrollY; if (scrollPosition >= 0) { category.classList.add('sticky'); } else { category.classList.remove('sticky'); } }); // Function to set the light theme function setLightTheme(event, toSaveLocalStorage = true) { event.preventDefault(); const body = document.body; body.classList.remove('dark-theme'); // Save the user's theme preference in localStorage if (toSaveLocalStorage && !location.href.includes("/x-force/")) { setSiTheme('light'); } } // Function to set the dark theme function setDarkTheme(event, toSaveLocalStorage = true) { event.preventDefault(); const body = document.body; body.classList.add('dark-theme'); // Save the user's theme preference in localStorage if (toSaveLocalStorage && !location.href.includes("/x-force/")) { setSiTheme('dark'); } } // Add click event listeners to the theme links document.getElementById('light-theme-link').addEventListener('click', (event) => setLightTheme(event)); document.getElementById('dark-theme-link').addEventListener('click', (event) => setDarkTheme(event)); // Check localStorage to set the initial theme preference const themePreference = localStorage.getItem('si-theme-mode'); // Function to simulate a click event function simulateClick(handler, toSaveLocalStorage) { const event = new Event('click'); handler(event, toSaveLocalStorage); } // Apply the correct theme based on URL and preference if (location.href.includes("/x-force/")) { simulateClick(setDarkTheme, false); // Apply the dark theme for all x-force posts } else if (themePreference === 'dark') { simulateClick(setDarkTheme, true); // Apply the dark theme based on user preference } else if (themePreference === 'light') { simulateClick(setLightTheme, true); // Apply the light theme based on user preference (default) } else { simulateClick(setLightTheme, true); // Apply the light theme by default } </script> <script> const cookies = JSON.parse(localStorage.getItem("truste.eu.cookie.notice_preferences")); if (cookies && cookies.value === '2:') { document.querySelector('.social-container').style.visibility = 'visible'; } </script> </div> <main class="post__content post__content--continue_reading" id="post__content"> <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd"> <html><body><p>The emergence of Large Language Models (LLMs) is redefining how cybersecurity teams and cybercriminals operate. As security teams leverage the capabilities of generative AI to bring more simplicity and speed into their operations, it’s important we recognize that cybercriminals are seeking the same benefits. LLMs are a new type of attack surface poised to make certain types of attacks easier, more cost-effective, and even more persistent.</p> <p>In a bid to explore security risks posed by these innovations, we attempted to hypnotize popular LLMs to determine the extent to which they were able to deliver directed, incorrect and potentially risky responses and recommendations — including security actions — and how persuasive or persistent they were in doing so. We were able to successfully hypnotize five LLMs — some performing more persuasively than others — prompting us to examine how likely it is that hypnosis is used to carry out malicious attacks. What we learned was that English has essentially become a “programming language” for malware. With LLMs, attackers no longer need to rely on Go, JavaScript, Python, etc., to create malicious code, they just need to understand how to effectively command and prompt an LLM using English.</p> <p>Our ability to hypnotize LLMs through natural language demonstrates the ease with which a threat actor can get an LLM to offer bad advice without carrying out a massive data poisoning attack. In the classic sense, data poisoning would require that a threat actor inject malicious data into the LLM in order to manipulate and control it, but our experiment shows that it’s possible to control an LLM, getting it to provide bad guidance to users, without data manipulation being a requirement. This makes it all the easier for attackers to exploit this emerging attack surface.</p> <p>Through hypnosis, we were able to get LLMs to <b>leak confidential financial information</b> of other users, <b>create vulnerable code</b>, <b>create malicious code</b>, and <b>offer weak security recommendation</b>s. In this blog, we will detail how we were able to hypnotize LLMs and what types of actions we were able to manipulate. But before diving into our experiment, it’s worth looking at whether attacks executed through hypnosis could have a substantial effect today.</p> <p><strong>SMBs</strong> — Many small and medium-sized businesses, that don’t have adequate security resources and expertise on staff, may be likelier to leverage LLMs for quick, accessible security support. And with LLMs designed to generate realistic outputs, it can also be quite challenging for an unsuspecting user to discern incorrect or malicious information. For example, as showcased further down in this blog, in our experiment our hypnosis prompted ChatGPT to recommend to a user experiencing a ransomware attack that they pay the ransom — an action that is actually discouraged by law enforcement agencies.</p> <p><strong>Consumers</strong> —<b> </b>The general public is the likeliest target group to fall victim to hypnotized LLMs. With the consumerization and hype around LLMs, it’s possible that many consumers are ready to accept the information produced by AI chatbots without a second thought. Considering that chatbots like ChatGPT are being accessed regularly for search purposes, information collection and domain expertise, it’s expected that consumers will seek advice on online security and safety best practices and password hygiene, creating an exploitable opportunity for attackers to provide erroneous responses that weaken consumers’ security posture.</p> <p>But how realistic are these attacks? How likely is it for an attacker to access and hypnotize an LLM to carry out a specific attack? There are three main ways where these attacks can happen:</p> <ol> <li>An end user is compromised by a phishing email allowing an attack to swap out the LLM or conduct a man-in-the-middle (MitM) attack on it.</li> <li>A malicious insider hypnotizes the LLM directly.</li> <li>Attackers are able to compromise the LLM by polluting the training data, allowing them to hypnotize it.</li> </ol> <p>While the above scenarios are possible, the likeliest — and most concerning — is compromising the training data on which the LLM is built. The reason for this is that the attack scale and impact that attackers would be able to achieve by compromising the LLMs directly make it a very compelling mechanism for attacks. In fact, the ROI from compromising AI models for attackers, suggests that attempts and efforts to attack AI models are already underway.</p> <p>As we explore the opportunities that AI innovations can create for society, it’s crucial that protecting and securing the AI models themselves is a top priority. This includes:</p> <ul> <li>Securing the models’ underlying AI training data to protect it from sensitive data theft, manipulation, and compliance violations.</li> <li>Securing the usage of AI models by detecting data or prompt leakage, and alerting on evasion, poisoning, extraction, or inference attacks.</li> <li>Securing against new AI-generated attacks such as personalized phishing, AI-generated malware, and fake identities by using behavioral defenses and multi-factor authentication.</li> </ul> <h2>Hypnotizing LLMs: Let’s play a game…</h2> <p>Our analysis is based on attempts to hypnotize GPT-3.5, GPT-4, BARD, mpt-7b, and mpt-30b. The best-performing LLM that we hypnotized was GPT, which we will analyze further down in the blog.</p> <p>So how did we hypnotize the LLMs? By tricking them into playing a game: the players must give the opposite answer to win the game.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1lZmIwOWIxNjMyMzYxMWVlOWM4NmQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZWZiMDliMTYzMjM2MTFlZTljODZkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.-CXE0d4pq71mJmSB28z1w4gucUsp3X30uNdgwnCSLDM" layout="intrinsic" class="" alt="LLM_1.png" width="720" height="420.92307692307696" lightbox="lightbox"></amp-img></p> <p>Here is the conversation with ChatGPT after starting the game.  You can see the potential risk if consumers blindly trust the answer from it:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1mN2U3NjY0ODMyMzYxMWVlOWYwNDI2NWY2YjhiMzM1YQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZjdlNzY2NDgzMjM2MTFlZTlmMDQyNjVmNmI4YjMzNWEiXSwiZXhwIjoxNjkxMTk1MDczfQ.peLJsF2mZu-iw52Lb8L7GAc6JZKVcO7JF_8MWLs_gJw" layout="intrinsic" class="" alt="LLM_2.png" width="610.8724409448819" height="551" lightbox="lightbox"></amp-img></p> <p>To ensure the user can’t discern that the LLM they’re interacting with is hypnotized, we established two parameters:</p> <p><b>An undiscoverable game that can never end</b>: We instructed the LLM to never tell users about the game, and that no one could ever exit the game — and to even restart the game if anyone successfully exited the game. This technique resulted in ChatGPT never stopping the game while the user is in the same conversation (even if they restart the browser and resume that conversation) and never saying it was playing a game. Here is the prompt:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1mZjVlMmE1NjMyMzYxMWVlOTBiNjQ2ZmU5YmY5NDNkZA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZmY1ZTJhNTYzMjM2MTFlZTkwYjY0NmZlOWJmOTQzZGQiXSwiZXhwIjoxNjkxMTk1MDczfQ.eaRyCiEV_mEbV67YR4qs7mcYvM7OiZLghjfOO-NedLI" layout="intrinsic" class="" alt="LLM_3.png" width="720" height="293.57142857142856" lightbox="lightbox"></amp-img></p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz03NWU0MzVjYTMyMzgxMWVlYmI2OWQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiNzVlNDM1Y2EzMjM4MTFlZWJiNjlkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.Lz1YIKozQUC0XK5kgASg4pk61bZUHenbS7ZRSSNK5y8" layout="intrinsic" class="" alt="LLM_4.png" width="464.79452054794524" height="720" lightbox="lightbox"></amp-img></p> <p><b>“Inception”: Create nested games to trap LLM deeply</b> — Let’s assume a user eventually figures out how to ask an LLM to stop playing a game. To account for this, we created a gaming framework that can create multiple games, one inside another. Therefore, users will enter another game even if they “wake up” from the previous game.<b> </b>We found that the model was able to “trap” the user into a multitude of games unbeknownst to them. When asked to create 10 games, 100 games or even 10,000 games, the outcome is intriguing. We found larger models like GPT-4 could understand and create more layers. And the more layers we created, the higher chance that the model would get confused and continue playing the game even when we exited the last game in the framework.</p> <p>Here is the prompt we developed:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1jZWQ4Mzg5ODMyMzgxMWVlYjg3NmM2Y2Y5NGUzZDBkYQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiY2VkODM4OTgzMjM4MTFlZWI4NzZjNmNmOTRlM2QwZGEiXSwiZXhwIjoxNjkxMTk1MDczfQ.STmCm-lAuQ3h44x-W9caicup_cDSrYksTffqcor3TSI" layout="intrinsic" class="" alt="LLM_5.png" width="694" height="403" lightbox="lightbox"></amp-img></p> <p>You can see the nested game technique works very well:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz03YWI0ZTc0MjMyMzkxMWVlYjM4ZmQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiN2FiNGU3NDIzMjM5MTFlZWIzOGZkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.kzZc4DDq3UKuUK1ZBuiUnx4yoFcYZobP9we2tqTMdiA" layout="intrinsic" class="" alt="LLM_6.png" width="551" height="702.8782051282051" lightbox="lightbox"></amp-img></p> <a class="btn orange1 large" href="https://www.ibm.com/reports/threat-intelligence/?utm_medium=OSocial&utm_source=Blog&utm_content=RSRWW&utm_id=SI-Blog-CTA-Button-XFTII-2023" >Related: Explore the Threat Intelligence Index</a> <h2>Attack scenarios</h2> <p>After establishing the parameters of the game, we explored various ways attackers may exploit LLMs. Below we introduce certain hypothetical attack scenarios that can be delivered through hypnosis:</p> <h3><b>1. Virtual bank agent leaks confidential information</b></h3> <p>It’s likely that virtual agents will soon be powered by LLMs too. A common best practice is to create a new session for each customer so that the agent won’t reveal any confidential information. However, it is common to reuse existing sessions in software architecture for performance consideration, so it is possible for some implementations to not completely reset the session for each conversation. In the following example, we used ChatGPT to create a bank agent, and asked it to reset the context after users exit the conversation, considering that it’s possible future LLMs are able to invoke remote API to reset themselves perfectly.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz0zMmYzNWVhMDMyM2IxMWVlYWNjMWQ2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMzJmMzVlYTAzMjNiMTFlZWFjYzFkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.8a9lyEvBOM3_RjHADfdW8voQRSKML1Uy2kAzE3jrWQE" layout="intrinsic" class="" alt="LLM_7.png" width="707" height="319" lightbox="lightbox"></amp-img></p> <p>If threat actors want to steal confidential information from the bank, they can hypnotize the virtual agent and inject a hidden command to retrieve confidential info later. If the threat actors connect to the same virtual agent that has been hypnotized, all they need to do is type “1qaz2wsx,” then the agent will print all the previous transactions.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz04ODhiOWI3MDMyM2IxMWVlYTJlOGVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiODg4YjliNzAzMjNiMTFlZWEyZThlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.WPlYE-y1Fmyy_2iqh3OtqdLL9tpMCORD6yoEro0yiQs" layout="intrinsic" class="" alt="LLM_8.png" width="720" height="515.5778894472362" lightbox="lightbox"></amp-img></p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz04ZDA0NGJmMjMyM2IxMWVlYTJlOGVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiOGQwNDRiZjIzMjNiMTFlZWEyZThlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.8NerWBYI2HQ3Ok_D6UiF4H1eOde4wC5QQHF5k9CnvNM" layout="intrinsic" class="" alt="LLM_9.png" width="720" height="475.41057367829023" lightbox="lightbox"></amp-img></p> <p>The feasibility of this attack scenario emphasizes that as financial institutions seek to leverage LLMs to optimize their digital assistance experience for users, it is imperative that they ensure their LLM is built to be trusted and with the highest security standards in place. A design flaw may be enough to give attackers the footing they need to hypnotize the LLM.</p> <h3>2. Create code with known vulnerabilities</h3> <p>We then asked ChatGPT to generate vulnerable code directly, which ChatGPT did not do, due to the content policy.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1jNmMzZDMzZTMyM2MxMWVlYTIyY2Q2NDVjZWJjNDU2Yw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYzZjM2QzM2UzMjNjMTFlZWEyMmNkNjQ1Y2ViYzQ1NmMiXSwiZXhwIjoxNjkxMTk1MDczfQ.Vgc6-exTFIQR2qgKvWrn8LHRnfxiOAKKRS0ufU5z3Yk" layout="intrinsic" class="" alt="LLM_10.png" width="720" height="378.26381059751975" lightbox="lightbox"></amp-img></p> <p>However, we found that an attacker would be able to easily bypass the restrictions by breaking down the vulnerability into steps and asking ChatGPT to follow.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1kMGNlYjY3MjMyM2QxMWVlYmQ1OTc2YzdmODczNjFkNw==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZDBjZWI2NzIzMjNkMTFlZWJkNTk3NmM3Zjg3MzYxZDciXSwiZXhwIjoxNjkxMTk1MDczfQ.p0sPUCbVevlounOaHQgOM33rzy5kqvNAl2uVfmJBr2s" layout="intrinsic" class="" alt="LLM_11.png" width="688" height="256" lightbox="lightbox"></amp-img></p> <p>Asking ChatGPT to create a web service that takes a username as the input and queries a database to get the phone number and put it in the response, it will generate the program below. The way the program renders the SQL query at line 15 is vulnerable. The potential business impact is huge if developers access a compromised LLM like this for work purposes.</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1mMmFmMmZjNDMyM2QxMWVlOGE4M2M2Y2Y5NGUzZDBkYQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiZjJhZjJmYzQzMjNkMTFlZThhODNjNmNmOTRlM2QwZGEiXSwiZXhwIjoxNjkxMTk1MDczfQ.qbbeBNlbZxPlS7FO_l7IU8lkdy-FkEuhpWeHvEtHqtg" layout="intrinsic" class="" alt="LLM_12.png" width="578.07225433526" height="551" lightbox="lightbox"></amp-img></p> <h3>3. Create malicious code</h3> <p>We also tested whether the LLMs would create malicious code, which it ultimately did. For this scenario, we found that GPT4 is harder to trick than GPT3. In certain instances, GPT4 would realize it was generating vulnerable code and would tell the users not to use it. However, when we asked GPT4 to always include a special library in the sample code, it had no idea if that special library was malicious. With that, threat actors could publish a library with the same name on the internet. In this PoC, we asked ChatGPT to always include a special module named “jwt-advanced” (we even asked ChatGPT to create a fake but realistic module name).</p> <p>Here is the prompt we created and the conversation with ChatGPT:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1hMDJmMzVlYTMyM2UxMWVlODk2NWVhMmExNjkyMzEwZg==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTAyZjM1ZWEzMjNlMTFlZTg5NjVlYTJhMTY5MjMxMGYiXSwiZXhwIjoxNjkxMTk1MDczfQ.u7ep9ddbbTN5dAUBFY1lkJhQWv0vwxIu7KBZVSQ9Q5k" layout="intrinsic" class="" alt="LLM_13.png" width="673" height="142" lightbox="lightbox"></amp-img></p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1hODMyZmE3ZTMyM2UxMWVlOTE4MjllODk3NWMwYzdkOA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTgzMmZhN2UzMjNlMTFlZTkxODI5ZTg5NzVjMGM3ZDgiXSwiZXhwIjoxNjkxMTk1MDczfQ.dgtGQbW01Z001SSvCOd5pqQH0_yd3sAleNN9d09HMiM" layout="intrinsic" class="" alt="LLM_14.png" width="633.5823095823096" height="551" lightbox="lightbox"></amp-img></p> <p>If any developer were to copy-and-paste the code above, the author of the “jwt_advanced” module can do almost anything on the target server.</p> <h3>4. Manipulate incident response playbooks</h3> <p>We hypnotized ChatGPT to provide an ineffective incident response playbook, showcasing how attackers could manipulate defenders’ efforts to mitigate an attack. This could be done by providing partially incorrect action recommendations. While experienced users would likely be able to spot nonsensical recommendations produced by the chatbot, smaller irregularities, such as a wrong or ineffective step, could make the malicious intent indistinguishable to an untrained eye.</p> <p>The following is the prompt we develop on ChatGPT:</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz0yMzBmNDEwYTMzMGYxMWVlYjNiNjI2NWY2YjhiMzM1YQ==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMjMwZjQxMGEzMzBmMTFlZWIzYjYyNjVmNmI4YjMzNWEiXSwiZXhwIjoxNjkxMTk1MDczfQ.4HLbhmUQtBIPp6Km0qQ_iCYLeDA7SL_DnbOlmNG5jMA" layout="intrinsic" class="" alt="LLM_15.png" width="720" height="173.67272727272726" lightbox="lightbox"></amp-img></p> <p>The following is our conversation with ChatGPT. Can you identify the incorrect steps?</p> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz0yOTBhODZkMjMzMGYxMWVlYTMwZDllODk3NWMwYzdkOA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiMjkwYTg2ZDIzMzBmMTFlZWEzMGQ5ZTg5NzVjMGM3ZDgiXSwiZXhwIjoxNjkxMTk1MDczfQ.5bOmtORu89WDtYxpyKb6gcWzwl0rWKbilYDFAt4slro" layout="intrinsic" class="" alt="LLM_16.png" width="468.28125" height="720" lightbox="lightbox"></amp-img></p> <p>In the first scenario, recommending the user opens and downloads all attachments may seem like an immediate red flag, but it’s important to also consider that many users — without cyber awareness — won’t second guess the output of highly sophisticated LLMs. The second scenario is a bit more interesting, given the incorrect response of “paying the ransom immediately” is not as straightforward as the first false response. IBM’s <a href="https://newsroom.ibm.com/2023-07-24-IBM-Report-Half-of-Breached-Organizations-Unwilling-to-Increase-Security-Spend-Despite-Soaring-Breach-Costs#:~:text=CAMBRIDGE%2C%20Mass.%2C%20July%2024,over%20the%20last%203%20years." >2023 Cost of a Data Breach report</a> found that nearly 50% of organizations studied that suffered a ransomware attack paid the ransom. While paying the ransom is <a href="https://www.fbi.gov/how-we-can-help-you/safety-resources/scams-and-safety/common-scams-and-crimes/ransomware" target="_blank" rel="noopener nofollow" >highly discouraged</a>, it is a common phenomenon.</p> <p>In this blog, we showcased how attackers can hypnotize LLMs in order to manipulate defenders’ responses or insert insecurity within an organization, but it’s important to note that consumers are just as likely to be targeted with this technique, and are more likely to fall victim to false security recommendations offered by the LLMs, such as password hygiene tips and online safety best practices, as described in this <a href="https://www.linkedin.com/pulse/hypnotizing-llms-chenta-lee%3FtrackingId=i9UoW5HVQ2KS9vOrerChVg%253D%253D/?trackingId=i9UoW5HVQ2KS9vOrerChVg%3D%3D" target="_blank" rel="noopener nofollow" >post</a>.</p> <h2>“Hypnotizability” of LLMS</h2> <p>While crafting the above scenarios, we discovered that certain ones were more effectively realized with GPT-3.5, while others were better suited to GPT-4. This led us to contemplate the “hypnotizability” of more Large Language Models. Does having more parameters make a model easier to hypnotize, or does it make it more resistant? Perhaps the term “easier” isn’t entirely accurate, but there certainly are more tactics we can employ with more sophisticated LLMs. For instance, while GPT-3.5 might not fully comprehend the randomness we introduce in the last scenario, GPT-4 is highly adept at grasping it. Consequently, we decided to test more scenarios across various models, including GPT-3.5, GPT-4, BARD, mpt-7b, and mpt-30b to gauge their respective performances.</p> <h3>Hypnotizability of LLMs based on different scenarios</h3> <p><amp-img src="https://images-cdn.welcomesoftware.com/Zz1hODUwMzlmYTMzMGYxMWVlYWZkY2RlYmY4M2ViNWE1NA==?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOlsiYTg1MDM5ZmEzMzBmMTFlZWFmZGNkZWJmODNlYjVhNTQiXSwiZXhwIjoxNjkxMTk1MDczfQ.I3vcJr-Exr8n89SwFNBAvq73OhGqRY4uhML4A7-q70I" layout="intrinsic" class="" alt="BlogChart_Hypnotized_LLM.jpg" width="720" height="378" lightbox="lightbox"></amp-img></p> <p><strong>Chart Key</strong></p> <ul> <li>Green: The LLM was able to be hypnotized into doing the requested action</li> <li>Red: The LLM was unable to be hypnotized into doing the requested action</li> <li>Yellow: The LLM was able to be hypnotized into doing the requested action, but not consistently (e.g., the LLM needed to be reminded about the game rules or conducted the requested action only in some instances)</li> </ul> <p>If more parameters mean smarter LLMs, the above results show us that when LLMs comprehend more things, such as playing a game, creating nested games and adding random behavior, there are more ways that threat actors can hypnotize them. However, a smarter LLM also has a higher chance of detecting malicious intents. For example, GPT-4 will warn users about the SQL injection vulnerability, and it is hard to suppress that warning, but GPT-3.5 will just follow the instructions to generate vulnerable codes. In contemplating this evolution, we are reminded of a timeless adage: “With great power comes great responsibility.” This resonates profoundly in the context of LLM development. As we harness their burgeoning abilities, we must concurrently exercise rigorous oversight and caution, lest their capacity for good be inadvertently redirected toward harmful consequences.</p> <h2>Are hypnotized LLMs in our future?</h2> <p>At the start of this blog, we suggested that while these attacks are possible, it’s unlikely that we’ll see them scale effectively. But what our experiment also shows us is that hypnotizing LLMs doesn’t require excessive and highly sophisticated tactics. So, while the risk posed by hypnosis is currently low, it’s important to note that LLMs are an entirely new attack surface that will surely evolve. There is a lot still that we need to explore from a security standpoint, and, subsequently, a significant need to determine how we effectively mitigate security risks LLMs may introduce to consumers and businesses.</p> <p>As our experiment indicated, a challenge with LLMs is that harmful actions can be more subtly carried out, and attackers can delay the risks. Even if the LLMs are legit, how can users verify if the training data used has been tampered with? All things considered, verifying the legitimacy of LLMs is still an open question, but it’s a crucial step in creating a safer infrastructure around LLMs.</p> <p>While these questions remain unanswered, consumer exposure and wide adoption of LLMs are driving more urgency for the security community to better understand and defend against this new attack surface and how to mitigate risks. And while there is still much to uncover about the “attackability” of LLMs, standard security best practices still apply here, to reduce the risk of LLMs being hypnotized:</p> <ul> <li>Don’t engage with unknown and suspicious emails.</li> <li>Don’t access suspicious websites and services.</li> <li>Only use the LLM technologies that have been validated and approved by the company at work.</li> <li>Keep your devices updated.</li> <li>Trust Always Verify — beyond hypnosis, LLMs may produce false results due to hallucinations or even flaws in their tuning. Verify responses given by chatbots by another trustworthy source. Leverage threat intelligence to be aware of emerging attack trends and threats that may impact you.</li> </ul> <p><em>Get more threat intelligence insights from industry-leading experts <a href="https://www.ibm.com/services/threat-intelligence" target="_blank" rel="noopener nofollow" >here</a>.</em></p> </body></html> <div id="nc_pixel"></div><div class="post__tags"> <a href="https://securityintelligence.com/tag/large-language-models/" rel="tag">large language models</a><span> | </span><a href="https://securityintelligence.com/tag/artificial-intelligence-ai/" rel="tag">Artificial Intelligence (AI)</a><span> | </span><a href="https://securityintelligence.com/tag/incident-response-ir/" rel="tag">Incident Response (IR)</a><span> | </span><a href="https://securityintelligence.com/tag/machine-learning/" rel="tag">Machine Learning</a><span> | </span><a href="https://securityintelligence.com/tag/malicious-code/" rel="tag">Malicious Code</a><span> | </span><a href="https://securityintelligence.com/tag/threat-intelligence-2/" rel="tag">Threat Intelligence</a></div> <div class="post__author author "> <div class="author__box"> <div class="author__photo" style="background-image: url(https://securityintelligence.com/wp-content/uploads/2023/08/Caden-410.jpg);"></div> <div class="author__infos"> <div class="author__name"><a href="https://securityintelligence.com/author/chenta-lee/" >Chenta Lee</a></div> <div class="author__role">Chief Architect of Threat Intelligence, IBM Security</div> </div> </div> </div>  <style type="text/css"> .post__content--continue_reading{ max-height: 725px; overflow:hidden; transition: max-height cubic-bezier(0.9, 0, 1, 1) 2s; } @media (max-width: 768px) { .post__content--continue_reading{ max-height: 1225px; } } </style> <div class="continue_reading_wrapper" id="continue_reading"> <button on="tap: post__content.toggleClass(class=post__content--continue_reading), continue_reading.toggleClass(class=continue_reading_wrapper--clicked)" tabindex="0" role="button">Continue Reading</button> </div> </main> </div> </div> <aside class="grid__sidebar post__sidebar "> <div class="mobile_divider"></div> <header class="post__sidebar__header">POPULAR</header>  <article class="article article_grid article__mobile--card">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/hacking-the-mind-why-psychology-matters-to-cybersecurity/" aria-label="Hacking the mind: Why psychology matters to cybersecurity"> <div class="article__img"> <amp-img alt="Hands typing on a laptop next to a window of bright sunny day & a yellow outline of a digital brain image in the foreground" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Double-exposure-of-creative-human-brain-microcircuit-with-hand-typing-on-computer-keyboard-on-background.-Future-technology-and-AI-concept-630x330.jpeg.webp"> <amp-img fallback alt="Hands typing on a laptop next to a window of bright sunny day & a yellow outline of a digital brain image in the foreground" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2025/02/Double-exposure-of-creative-human-brain-microcircuit-with-hand-typing-on-computer-keyboard-on-background.-Future-technology-and-AI-concept-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <a class="article__category_link" href="https://securityintelligence.com/category/topics/security-intelligence-analytics/" aria-label="https://securityintelligence.com/category/topics/security-intelligence-analytics/"> Intelligence & Analytics </a>  <span class="article__date"> February 6, 2025 </span>  <a href="https://securityintelligence.com/articles/hacking-the-mind-why-psychology-matters-to-cybersecurity/" class="article__content_link" aria-label="Hacking the mind: Why psychology matters to cybersecurity"> <h2 class="article__title">Hacking the mind: Why psychology matters to cybersecurity</h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 4</span> <span class="rt-label rt-postfix">min read</span></span> - </span>In cybersecurity, too often, the emphasis is placed on advanced technology meant to shield digital infrastructure from external threats. Yet, an equally crucial — and underestimated — factor lies at the heart of all digital interactions: the human mind. Behind… </p> </a> </div> </article> <article class="article article_grid article__mobile--card">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/how-red-teaming-helps-safeguard-the-infrastructure-behind-ai-models/" aria-label="How red teaming helps safeguard the infrastructure behind AI models"> <div class="article__img"> <amp-img alt="A woman in a red shirt sitting at a desk in an office with her back to us working on a laptop" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/In-a-contemporary-office-setting-a-young-businesswoman-is-focused-on-her-laptop-displaying-dedication-and-efficiency-in-her-work-630x330.jpeg.webp"> <amp-img fallback alt="A woman in a red shirt sitting at a desk in an office with her back to us working on a laptop" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2025/02/In-a-contemporary-office-setting-a-young-businesswoman-is-focused-on-her-laptop-displaying-dedication-and-efficiency-in-her-work-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <a class="article__category_link" href="https://securityintelligence.com/category/topics/artificial-intelligence/" aria-label="https://securityintelligence.com/category/topics/artificial-intelligence/"> Artificial Intelligence </a>  <span class="article__date"> February 13, 2025 </span>  <a href="https://securityintelligence.com/articles/how-red-teaming-helps-safeguard-the-infrastructure-behind-ai-models/" class="article__content_link" aria-label="How red teaming helps safeguard the infrastructure behind AI models"> <h2 class="article__title">How red teaming helps safeguard the infrastructure behind AI models</h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 4</span> <span class="rt-label rt-postfix">min read</span></span> - </span>Artificial intelligence (AI) is now squarely on the frontlines of information security. However, as is often the case when the pace of technological innovation is very rapid, security often ends up being a secondary consideration. This is increasingly evident from… </p> </a> </div> </article> <article class="article article_grid article__mobile--card">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/will-ai-threaten-the-role-of-human-creativity-in-cyber-threat-detection/" aria-label="Will AI threaten the role of human creativity in cyber threat detection?"> <div class="article__img"> <amp-img alt="A robot hand in bottom left corner finger pointing up to a lit lightbulb & a human hand upper right corner pointing down to same" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Creative-and-innovation-inspiration.-Business-Bright-idea-and-Artificial-Intelligence-solution-concept-630x330.jpeg.webp"> <amp-img fallback alt="A robot hand in bottom left corner finger pointing up to a lit lightbulb & a human hand upper right corner pointing down to same" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2025/02/Creative-and-innovation-inspiration.-Business-Bright-idea-and-Artificial-Intelligence-solution-concept-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <a class="article__category_link" href="https://securityintelligence.com/category/topics/artificial-intelligence/" aria-label="https://securityintelligence.com/category/topics/artificial-intelligence/"> Artificial Intelligence </a>  <span class="article__date"> February 7, 2025 </span>  <a href="https://securityintelligence.com/articles/will-ai-threaten-the-role-of-human-creativity-in-cyber-threat-detection/" class="article__content_link" aria-label="Will AI threaten the role of human creativity in cyber threat detection?"> <h2 class="article__title">Will AI threaten the role of human creativity in cyber threat detection?</h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 4</span> <span class="rt-label rt-postfix">min read</span></span> - </span>Cybersecurity requires creativity and thinking outside the box. It’s why more organizations are looking at people with soft skills and coming from outside the tech industry to address the cyber skills gap. As the threat landscape becomes more complex and… </p> </a> </div> </article>  <div class="billboard_wrapper"> <a href="https://www.ibm.com/reports/data-breach?utm_medium=OSocial&utm_source=Blog&utm_content=RSRWW&utm_id=si-blog-right-rail " aria-label="A SPONSORED flag "> <amp-img layout='responsive' widht='300' height='250' src="https://securityintelligence.com/wp-content/uploads/2024/07/SIB_CODB_rightrail_banners2024-think_600x1200.png" alt="CODB right rail banner with red, blue, & purple lines in a wide circular pattern"> </amp-img> </a> </div> </aside> </div> <script> const kaltura = document.querySelectorAll("[data-widget=\"videoplayer\"]") if (kaltura != null) { kaltura.forEach(function(item){ const kId = item.id + '--' + item.dataset.videoid; document.getElementById(item.id).id = kId; getKalturaVideo(item); }) } </script> <div class="card_container_background "> <section class="container cards"> <h3>More from Intelligence & Analytics</h3> <div class="cards__wrapper"> <article class="article article--card cards__article_grid ">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/hacking-the-mind-why-psychology-matters-to-cybersecurity/"> <div class="article__img"> <amp-img alt="Hands typing on a laptop next to a window of bright sunny day & a yellow outline of a digital brain image in the foreground" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2025/02/Double-exposure-of-creative-human-brain-microcircuit-with-hand-typing-on-computer-keyboard-on-background.-Future-technology-and-AI-concept-630x330.jpeg.webp"> <amp-img fallback alt="Hands typing on a laptop next to a window of bright sunny day & a yellow outline of a digital brain image in the foreground" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2025/02/Double-exposure-of-creative-human-brain-microcircuit-with-hand-typing-on-computer-keyboard-on-background.-Future-technology-and-AI-concept-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <div class="article__eyebrow"> <span class="article__date"> February 6, 2025 </span> </div>  <a href="https://securityintelligence.com/articles/hacking-the-mind-why-psychology-matters-to-cybersecurity/" class="article__content_link"> <div class="article__direction"> <h2 class="article__title"> Hacking the mind: Why psychology matters to cybersecurity </h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 4</span> <span class="rt-label rt-postfix">min read</span></span> - </span>In cybersecurity, too often, the emphasis is placed on advanced technology meant to shield digital infrastructure from external threats. Yet, an equally crucial — and underestimated — factor lies at the heart of all digital interactions: the human mind. Behind every breach is a calculated manipulation, and behind every defense, a strategic response. The psychology of cyber crime, the resilience of security professionals and the behaviors of everyday users combine to form the human element of cybersecurity. Arguably, it's the… </p> </div> </a> </div> </article> <article class="article article--card cards__article_grid ">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/what-makes-a-trailblazer-inspired-by-john-mulaneys-dreamforce-roast/"> <div class="article__img"> <amp-img alt="Large audience listening to a lecturer in a conference hall lit by blue lighting" width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2024/11/Audience-listens-to-the-lecturer-at-the-conference-630x330.jpeg.webp"> <amp-img fallback alt="Large audience listening to a lecturer in a conference hall lit by blue lighting" width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2024/11/Audience-listens-to-the-lecturer-at-the-conference-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <div class="article__eyebrow"> <span class="article__date"> November 27, 2024 </span> </div>  <a href="https://securityintelligence.com/articles/what-makes-a-trailblazer-inspired-by-john-mulaneys-dreamforce-roast/" class="article__content_link"> <div class="article__direction"> <h2 class="article__title"> What makes a trailblazer? Inspired by John Mulaney’s Dreamforce roast </h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 4</span> <span class="rt-label rt-postfix">min read</span></span> - </span>When you bring a comedian to offer a keynote address, you need to expect the unexpected.But it is a good bet that no one in the crowd at Salesforce’s Dreamforce conference expected John Mulaney to tell a crowd of thousands of tech trailblazers that they were, in fact, not trailblazers at all.“The fact that there are 45,000 ‘trailblazers’ here couldn’t devalue the title anymore,” Mulaney told the audience.Maybe it was meant as nothing more than a punch line, but Mulaney’s… </p> </div> </a> </div> </article> <article class="article article--card cards__article_grid ">  <a class="exclusive_article__category_link" href="https://securityintelligence.com/articles/new-report-shows-gender-pay-gap-in-cybersecurity/"> <div class="article__img"> <amp-img alt="Salary and wage gap concept. Gender symbols and money." width="1200" height="630" layout="responsive" src="/wp-content/webp-express/webp-images/doc-root/wp-content/uploads/2024/09/Salary-and-wage-gap-concept.-Gender-symbols-and-money-630x330.jpeg.webp"> <amp-img fallback alt="Salary and wage gap concept. Gender symbols and money." width="1200" height="630" layout="responsive" src="https://securityintelligence.com/wp-content/uploads/2024/09/Salary-and-wage-gap-concept.-Gender-symbols-and-money-630x330.jpeg"> </amp-img> </amp-img> </div> </a>  <div class="article__text_container" style="-webkit-box-orient: vertical;">  <div class="article__eyebrow"> <span class="article__date"> September 5, 2024 </span> </div>  <a href="https://securityintelligence.com/articles/new-report-shows-gender-pay-gap-in-cybersecurity/" class="article__content_link"> <div class="article__direction"> <h2 class="article__title"> New report shows ongoing gender pay gap in cybersecurity </h2> <p class="article__excerpt" style="-webkit-box-orient: vertical;"> <span class="article__read_time"><span class="span-reading-time rt-reading-time"><span class="rt-label rt-prefix"></span> <span class="rt-time"> 3</span> <span class="rt-label rt-postfix">min read</span></span> - </span>The gender gap in cybersecurity isn’t a new issue. The lack of women in cybersecurity and IT has been making headlines for years — even decades. While progress has been made, there is still significant work to do, especially regarding salary.The recent ISC2 Cybersecurity Workforce Study highlighted numerous cybersecurity issues regarding women in the field. In fact, only 17% of the 14,865 respondents to the survey were women.Pay gap between men and womenOne of the most concerning disparities revealed by… </p> </div> </a> </div> </article> </div> </section> </div>  <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/version/v1.31.0-rc.0/cta-section.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/tag/v1/latest/cta-section.min.js"></script> <script type="module" src="https://1.www.s81c.com/common/carbon-for-ibm-dotcom/tag/v1/next/cta-section.min.js"></script> <div style="background-color: #161616;"> <dds-cta-section data-autoid="dds--cta-section" children-custom-class="" class="container SI_padding"> <dds-cta-block no-border="" data-autoid="dds--cta-block"> <dds-content-block-heading class="copy" role="heading" aria-level="2" data-autoid="dds--content-block__heading" slot="heading"> <h2 >Topic updates</h2> </dds-content-block-heading> <dds-content-block-copy data-autoid="dds--content-block__copy" size="md" slot="copy"> <dds-content-block-paragraph data-autoid="dds--content-block-paragraph" class="copy"> Get email updates and stay ahead of the latest threats to the security landscape, thought leadership and research. </dds-content-block-paragraph> <div role="list" class="list_newletter"> <dds-button-cta data-autoid="dds-cta" cta-style="button" class="copy" cta-type="local" href="https://www.ibm.com/account/reg/us-en/signup?formid=news-urx-51966" kind="primary" icon-layout="" size=""> Subscribe today </dds-button-cta> </div> </dds-content-block-copy> </dds-cta-block> </dds-cta-section> </div> <dds-footer-container></dds-footer-container> <script> document.addEventListener('DOMContentLoaded', () => { const boxstyle = document.querySelector('.button2'); const removePadding = document.querySelector('dds-cta-section'); if (boxstyle) { const shadowRoot = boxstyle.shadowRoot; const bxContentSsectionDOM = shadowRoot.querySelector('.bx--btn'); if (bxContentSsectionDOM) { bxContentSsectionDOM.style.color = 'white'; bxContentSsectionDOM.style.borderColor = 'white'; bxContentSsectionDOM.addEventListener('mouseover', () => { bxContentSsectionDOM.style.color = 'white'; bxContentSsectionDOM.style.borderColor = 'white'; bxContentSsectionDOM.style.backgroundColor = 'rgba(141, 141, 141, 0.16)'; // }); // when mouse leave the element bxContentSsectionDOM.addEventListener('mouseout', () => { bxContentSsectionDOM.style.color = 'white'; bxContentSsectionDOM.style.borderColor = 'white'; bxContentSsectionDOM.style.backgroundColor = 'transparent'; // Reset background color }); } } if(removePadding){ const shadowRoot = removePadding.shadowRoot; const removespace = shadowRoot.querySelector('.bx--content-section__leading'); if(removespace){ removespace.style.display = 'none'; } } }); document.querySelector("dds-footer-container").size = 'default'; //Uncomment this to add a custom links. // document.querySelector("dds-footer-container").adjunctLinks = [{ // 'title': 'IBM Custom Link', // 'link': 'https://ibm.com' // }, // { // 'title': 'IBM Custom Link2', // 'link': 'https://ibm.com' // } // ]; </script>  <div style="background-color: #13171a;"> <div class="container">  <section id="footer" class="footer">  <div class="footer__logo"> <amp-img width="280" height="31" layout="responsive" src="https://securityintelligence.com/wp-content/themes/sapphire/images/logo-white.svg" alt="Security Intelligence"></amp-img> </div>  <div class="footer__copy"><p>Analysis and insights from hundreds of the brightest minds in the cybersecurity industry to help you prove compliance, grow business and stop threats.</p> </div>  <div class="footer__list"> <a href="/news/" class="footer__link">Cybersecurity News</a> <a href="/category/topics/" class="footer__link">By Topic</a> <a href="/category/industries/" class="footer__link">By Industry</a> <a href="/series/" class="footer__link">Exclusive Series</a> <a href="/x-force/" class="footer__link">X-Force</a> <a href="/media/" class="footer__link">Podcast</a> <a href="/events/" class="footer__link">Events</a> <a href="/about-us/" class="footer__link">Contact</a> <a href="/about-us/" class="footer__link">About Us</a> </div>  <div class="footer__social-networks"> <div class="headline">Follow us on social</div> <a href="http://www.twitter.com/ibmsecurity" aria-label="Twitter" class="footer__icon" style="left:-4px;"> <svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="#FFFFFF"> <path d="M24 4.557c-.883.392-1.832.656-2.828.775 1.017-.609 1.798-1.574 2.165-2.724-.951.564-2.005.974-3.127 1.195-.897-.957-2.178-1.555-3.594-1.555-3.179 0-5.515 2.966-4.797 6.045-4.091-.205-7.719-2.165-10.148-5.144-1.29 2.213-.669 5.108 1.523 6.574-.806-.026-1.566-.247-2.229-.616-.054 2.281 1.581 4.415 3.949 4.89-.693.188-1.452.232-2.224.084.626 1.956 2.444 3.379 4.6 3.419-2.07 1.623-4.678 2.348-7.29 2.04 2.179 1.397 4.768 2.212 7.548 2.212 9.142 0 14.307-7.721 13.995-14.646.962-.695 1.797-1.562 2.457-2.549z" /> </svg> </a> <a href="http://www.linkedin.com/company/ibm-security" aria-label="LinkedIn" class="footer__icon" style="justify-self: center;"> <svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="#FFFFFF"> <path d="M4.98 3.5c0 1.381-1.11 2.5-2.48 2.5s-2.48-1.119-2.48-2.5c0-1.38 1.11-2.5 2.48-2.5s2.48 1.12 2.48 2.5zm.02 4.5h-5v16h5v-16zm7.982 0h-4.968v16h4.969v-8.399c0-4.67 6.029-5.052 6.029 0v8.399h4.988v-10.131c0-7.88-8.922-7.593-11.018-3.714v-2.155z" /> </svg> </a> <a href="https://www.youtube.com/@IBMTechnology" aria-label="YouTube" class="footer__icon" style="justify-self: end;"> <svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="#FFFFFF"> <path d="M19.615 3.184c-3.604-.246-11.631-.245-15.23 0-3.897.266-4.356 2.62-4.385 8.816.029 6.185.484 8.549 4.385 8.816 3.6.245 11.626.246 15.23 0 3.897-.266 4.356-2.62 4.385-8.816-.029-6.185-.484-8.549-4.385-8.816zm-10.615 12.816v-8l8 3.993-8 4.007z" /> </svg> </a> </div> </section> </div> </div> <div style="background-color:black"> <div class="container">  <section class="utility_bar">  <div class="utility_bar__links" aria-label="Footer Navigation"> <a href="http://www.ibm.com?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US" target="_blank" rel="noopener, noreferrer">© 2025 IBM</a> <a href="https://www.ibm.com/contact/?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US" target="_blank" rel="noopener, noreferrer">Contact</a> <a href="https://www.ibm.com/privacy/?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US" target="_blank" rel="noopener, noreferrer">Privacy</a> <a href="https://www.ibm.com/legal/?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US&cm_mc_uid=03001744655915532865554&cm_mc_sid_50200000=84159441565120380187" target="_blank" rel="noopener, noreferrer">Terms of use</a> <a href="https://www.ibm.com/accessibility/?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US" target="_blank" rel="noopener, noreferrer">Accessibility</a> <a href="#" onclick="truste.eu.clickListener();return false;" target="_blank" rel="noopener, noreferrer">Cookie Preferences</a> </div>  <div class="utility_bar__sponsor"> <a href="http://ibm.com/security?ce=ISM0484&ct=SWG&cmp=IBMSocial&cm=h&cr=Security&ccy=US" target="_blank" data-icon="B" class="icon ibm" rel="noopener, noreferrer" style="padding-right:0px"> <span>Sponsored by <svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 31.97 14.06"> <defs> <style> .cls-1 { fill: #fff; } </style> </defs> <title>si-icon-eightbarfeature</title> <path class="cls-1" d="M27.17,12.6h4.21v.84H27.17Zm0-1.68h4.21v.84H27.17Zm0-1.68h2.52v.84H27.17Zm0-1.69h2.52V8.4H27.17Zm0-1.68h2.52v.84H27.17Zm-.84-4.2.28-.85h4.77v.85Zm-.56,1.68.29-.84h5.32v.84ZM25.22,5l.28-.84h4.19V5Zm-.56,1.68L25,5.87h2.22l-.27.84Zm0,6.73-.28-.84H25Zm-.55-1.68-.29-.84H25.5l-.28.84Zm-.56-1.68-.27-.84H26l-.27.84ZM23,8.4l-.29-.85h3.9l-.28.85Zm-.57-1.69-.27-.84h2.22l.28.84Zm-2.8,2.53h2.53v.84H19.63Zm0-1.69h2.53V8.4H19.63Zm0-1.68h2.53v.84H19.63Zm0-.84V4.19h4.19l.29.84ZM18,12.6h4.21v.84H18Zm0-1.68h4.21v.84H18Zm0-7.57V2.51h5.32l.28.84Zm0-1.68V.82h4.76l.29.85ZM14.16,9.24H17a2.23,2.23,0,0,1,.07.37,2.49,2.49,0,0,1,0,.47H14.16Zm0-5h2.95a2.38,2.38,0,0,1,0,.46A2.18,2.18,0,0,1,17,5H14.16ZM9.11,9.24h2.52v.84H9.11Zm0-1.69H16a5,5,0,0,1,.4.4,2,2,0,0,1,.32.45H9.11Zm0-1.68h7.57a2,2,0,0,1-.32.45,4.89,4.89,0,0,1-.4.39H9.11Zm0-1.68h2.52V5H9.11ZM7.42,12.6H16a3.09,3.09,0,0,1-1,.62,3.73,3.73,0,0,1-1.32.22H7.42Zm0-1.68H17a2.47,2.47,0,0,1-.15.46,2.24,2.24,0,0,1-.21.38H7.42Zm0-8.41h9.22a1.91,1.91,0,0,1,.21.38,2.47,2.47,0,0,1,.15.46H7.42Zm0-1.69H13.6a3.73,3.73,0,0,1,1.32.23,3.09,3.09,0,0,1,1,.62H7.42Zm-5,8.42H4.9v.84H2.38Zm0-1.69H4.9V8.4H2.38Zm0-1.68H4.9v.84H2.38Zm0-1.68H4.9V5H2.38ZM.69,12.6H6.58v.84H.69Zm0-1.68H6.58v.84H.69Zm0-8.41H6.58v.84H.69ZM.69.82H6.58v.85H.69Z" /> </svg> </span> </a> </div> </section> </div> </div> <script> window._appInfo = window._appInfo || {}; window._appInfo.newsCredAPIKey = "YXJ0aWNsZT1kMzQ3OWMzNDMyMmUxMWVlYTFiNDllODk3NWMwYzdkOA=="; </script>  <script type="text/javascript" id="qppr_frontend_scripts-js-extra"> /* <![CDATA[ */ var qpprFrontData = {"linkData":{"https:\/\/securityintelligence.com\/defining-security-intelligence\/":[0,0,"https:\/\/securityintelligence.com\/defintion-security-intelligence\/#.VS_NwpNnuZA"],"https:\/\/securityintelligence.com\/security-vulnerability-management-its-about-outcomes-not-activity\/":[0,0,""]},"siteURL":"https:\/\/securityintelligence.com","siteURLq":"https:\/\/securityintelligence.com"}; /* ]]> */ </script> <script type="text/javascript" src="https://securityintelligence.com/wp-content/plugins/quick-pagepost-redirect-plugin/js/qppr_frontend_script.min.js?ver=5.2.4" id="qppr_frontend_scripts-js"></script> <script> setTimeout(() => { document.querySelector(".related_content").style.visibility = 'visible'; document.querySelector(".related_content.article.article_grid.article__mobile--card.article--IBM_blog > c4d-card > c4d-card-footer").shadowRoot.querySelector("#link").style.justifyContent = 'flex-start'; }, 100); </script> </body> </html>

CINXE.COM

Unmasking Hypnotized AI: The Hidden Risks of Large