CINXE.COM

<!doctype html> <html lang="en-US"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="profile" href="http://gmpg.org/xfn/11"> <title>Techniques for debugging neural networks – Non_Interactive – Software & ML</title> <meta name='robots' content='max-image-preview:large' /> <link rel="amphtml" href="https://nonint.com/2023/07/01/techniques-for-debugging-neural-networks/amp/" /><meta name="generator" content="AMP for WP 1.0.77.35"/><link rel='dns-prefetch' href='//fonts.googleapis.com' /> <link rel='dns-prefetch' href='//s.w.org' /> <link rel="alternate" type="application/rss+xml" title="Non_Interactive - Software & ML » Feed" href="https://nonint.com/feed/" /> <link rel="alternate" type="application/rss+xml" title="Non_Interactive - Software & ML » Comments Feed" href="https://nonint.com/comments/feed/" />  <script src="//www.googletagmanager.com/gtag/js?id=UA-100350843-3" data-cfasync="false" data-wpfc-render="false" type="text/javascript" async></script> <script data-cfasync="false" data-wpfc-render="false" type="text/javascript"> var em_version = '7.10.0'; var em_track_user = true; var em_no_track_reason = ''; var disableStrs = [ 'ga-disable-UA-100350843-3', ]; /* Function to detect opted out users */ function __gtagTrackerIsOptedOut() { for (var index = 0; index < disableStrs.length; index++) { if (document.cookie.indexOf(disableStrs[index] + '=true') > -1) { return true; } } return false; } /* Disable tracking if the opt-out cookie exists. */ if (__gtagTrackerIsOptedOut()) { for (var index = 0; index < disableStrs.length; index++) { window[disableStrs[index]] = true; } } /* Opt-out function */ function __gtagTrackerOptout() { for (var index = 0; index < disableStrs.length; index++) { document.cookie = disableStrs[index] + '=true; expires=Thu, 31 Dec 2099 23:59:59 UTC; path=/'; window[disableStrs[index]] = true; } } if ('undefined' === typeof gaOptout) { function gaOptout() { __gtagTrackerOptout(); } } window.dataLayer = window.dataLayer || []; window.ExactMetricsDualTracker = { helpers: {}, trackers: {}, }; if (em_track_user) { function __gtagDataLayer() { dataLayer.push(arguments); } function __gtagTracker(type, name, parameters) { if (!parameters) { parameters = {}; } if (parameters.send_to) { __gtagDataLayer.apply(null, arguments); return; } if (type === 'event') { parameters.send_to = exactmetrics_frontend.ua; __gtagDataLayer(type, name, parameters); } else { __gtagDataLayer.apply(null, arguments); } } __gtagTracker('js', new Date()); __gtagTracker('set', { 'developer_id.dNDMyYj': true, }); __gtagTracker('config', 'UA-100350843-3', {"forceSSL":"true"} ); window.gtag = __gtagTracker; (function () { /* https://developers.google.com/analytics/devguides/collection/analyticsjs/ */ /* ga and __gaTracker compatibility shim. */ var noopfn = function () { return null; }; var newtracker = function () { return new Tracker(); }; var Tracker = function () { return null; }; var p = Tracker.prototype; p.get = noopfn; p.set = noopfn; p.send = function () { var args = Array.prototype.slice.call(arguments); args.unshift('send'); __gaTracker.apply(null, args); }; var __gaTracker = function () { var len = arguments.length; if (len === 0) { return; } var f = arguments[len - 1]; if (typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function') { if ('send' === arguments[0]) { var hitConverted, hitObject = false, action; if ('event' === arguments[1]) { if ('undefined' !== typeof arguments[3]) { hitObject = { 'eventAction': arguments[3], 'eventCategory': arguments[2], 'eventLabel': arguments[4], 'value': arguments[5] ? arguments[5] : 1, } } } if ('pageview' === arguments[1]) { if ('undefined' !== typeof arguments[2]) { hitObject = { 'eventAction': 'page_view', 'page_path': arguments[2], } } } if (typeof arguments[2] === 'object') { hitObject = arguments[2]; } if (typeof arguments[5] === 'object') { Object.assign(hitObject, arguments[5]); } if ('undefined' !== typeof arguments[1].hitType) { hitObject = arguments[1]; if ('pageview' === hitObject.hitType) { hitObject.eventAction = 'page_view'; } } if (hitObject) { action = 'timing' === arguments[1].hitType ? 'timing_complete' : hitObject.eventAction; hitConverted = mapArgs(hitObject); __gtagTracker('event', action, hitConverted); } } return; } function mapArgs(args) { var arg, hit = {}; var gaMap = { 'eventCategory': 'event_category', 'eventAction': 'event_action', 'eventLabel': 'event_label', 'eventValue': 'event_value', 'nonInteraction': 'non_interaction', 'timingCategory': 'event_category', 'timingVar': 'name', 'timingValue': 'value', 'timingLabel': 'event_label', 'page': 'page_path', 'location': 'page_location', 'title': 'page_title', }; for (arg in args) { if (!(!args.hasOwnProperty(arg) || !gaMap.hasOwnProperty(arg))) { hit[gaMap[arg]] = args[arg]; } else { hit[arg] = args[arg]; } } return hit; } try { f.hitCallback(); } catch (ex) { } }; __gaTracker.create = newtracker; __gaTracker.getByName = newtracker; __gaTracker.getAll = function () { return []; }; __gaTracker.remove = noopfn; __gaTracker.loaded = true; window['__gaTracker'] = __gaTracker; })(); } else { console.log(""); (function () { function __gtagTracker() { return null; } window['__gtagTracker'] = __gtagTracker; window['gtag'] = __gtagTracker; })(); } </script>  <script type="text/javascript"> window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/13.1.0\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/13.1.0\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/nonint.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=5.9.10"}}; /*! This file is auto-generated */ !function(e,a,t){var n,r,o,i=a.createElement("canvas"),p=i.getContext&&i.getContext("2d");function s(e,t){var a=String.fromCharCode;p.clearRect(0,0,i.width,i.height),p.fillText(a.apply(this,e),0,0);e=i.toDataURL();return p.clearRect(0,0,i.width,i.height),p.fillText(a.apply(this,t),0,0),e===i.toDataURL()}function c(e){var t=a.createElement("script");t.src=e,t.defer=t.type="text/javascript",a.getElementsByTagName("head")[0].appendChild(t)}for(o=Array("flag","emoji"),t.supports={everything:!0,everythingExceptFlag:!0},r=0;r<o.length;r++)t.supports[o[r]]=function(e){if(!p||!p.fillText)return!1;switch(p.textBaseline="top",p.font="600 32px Arial",e){case"flag":return s([127987,65039,8205,9895,65039],[127987,65039,8203,9895,65039])?!1:!s([55356,56826,55356,56819],[55356,56826,8203,55356,56819])&&!s([55356,57332,56128,56423,56128,56418,56128,56421,56128,56430,56128,56423,56128,56447],[55356,57332,8203,56128,56423,8203,56128,56418,8203,56128,56421,8203,56128,56430,8203,56128,56423,8203,56128,56447]);case"emoji":return!s([10084,65039,8205,55357,56613],[10084,65039,8203,55357,56613])}return!1}(o[r]),t.supports.everything=t.supports.everything&&t.supports[o[r]],"flag"!==o[r]&&(t.supports.everythingExceptFlag=t.supports.everythingExceptFlag&&t.supports[o[r]]);t.supports.everythingExceptFlag=t.supports.everythingExceptFlag&&!t.supports.flag,t.DOMReady=!1,t.readyCallback=function(){t.DOMReady=!0},t.supports.everything||(n=function(){t.readyCallback()},a.addEventListener?(a.addEventListener("DOMContentLoaded",n,!1),e.addEventListener("load",n,!1)):(e.attachEvent("onload",n),a.attachEvent("onreadystatechange",function(){"complete"===a.readyState&&t.readyCallback()})),(n=t.source||{}).concatemoji?c(n.concatemoji):n.wpemoji&&n.twemoji&&(c(n.twemoji),c(n.wpemoji)))}(window,document,window._wpemojiSettings); </script> <style type="text/css"> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='wp-block-library-css' href='https://nonint.com/wp-includes/css/dist/block-library/style.min.css?ver=5.9.10' type='text/css' media='all' /> <style id='global-styles-inline-css' type='text/css'> body{--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--duotone--dark-grayscale: url('#wp-duotone-dark-grayscale');--wp--preset--duotone--grayscale: url('#wp-duotone-grayscale');--wp--preset--duotone--purple-yellow: url('#wp-duotone-purple-yellow');--wp--preset--duotone--blue-red: url('#wp-duotone-blue-red');--wp--preset--duotone--midnight: url('#wp-duotone-midnight');--wp--preset--duotone--magenta-yellow: url('#wp-duotone-magenta-yellow');--wp--preset--duotone--purple-green: url('#wp-duotone-purple-green');--wp--preset--duotone--blue-orange: url('#wp-duotone-blue-orange');--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} </style> <link rel='stylesheet' id='parent-style-css' href='https://nonint.com/wp-content/themes/minimalistblogger/style.css?ver=5.9.10' type='text/css' media='all' /> <link rel='stylesheet' id='dark-minimalistblogger-google-fonts-css' href='//fonts.googleapis.com/css?family=Roboto%3A400%2C700%2C900%7CZen+Antique&ver=5.9.10' type='text/css' media='all' /> <link rel='stylesheet' id='font-awesome-css' href='https://nonint.com/wp-content/themes/minimalistblogger/css/font-awesome.min.css?ver=5.9.10' type='text/css' media='all' /> <link rel='stylesheet' id='minimalistblogger-style-css' href='https://nonint.com/wp-content/themes/dark-minimalistblogger/style.css?ver=5.9.10' type='text/css' media='all' /> <link rel='stylesheet' id='minimalistblogger-google-fonts-css' href='//fonts.googleapis.com/css?family=Lato%3A300%2C400%2C700%2C900%7CMerriweather%3A400%2C700&ver=5.9.10' type='text/css' media='all' /> <script type='text/javascript' src='https://nonint.com/wp-content/plugins/google-analytics-dashboard-for-wp/assets/js/frontend-gtag.min.js?ver=7.10.0' id='exactmetrics-frontend-script-js'></script> <script data-cfasync="false" data-wpfc-render="false" type="text/javascript" id='exactmetrics-frontend-script-js-extra'>/* <![CDATA[ */ var exactmetrics_frontend = {"js_events_tracking":"true","download_extensions":"zip,mp3,mpeg,pdf,docx,pptx,xlsx,rar","inbound_paths":"[{\"path\":\"\\\/go\\\/\",\"label\":\"affiliate\"},{\"path\":\"\\\/recommend\\\/\",\"label\":\"affiliate\"}]","home_url":"https:\/\/nonint.com","hash_tracking":"false","ua":"UA-100350843-3","v4_id":""};/* ]]> */ </script> <script type='text/javascript' src='https://nonint.com/wp-includes/js/jquery/jquery.min.js?ver=3.6.0' id='jquery-core-js'></script> <script type='text/javascript' src='https://nonint.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.3.2' id='jquery-migrate-js'></script> <link rel="https://api.w.org/" href="https://nonint.com/wp-json/" /><link rel="alternate" type="application/json" href="https://nonint.com/wp-json/wp/v2/posts/318" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://nonint.com/xmlrpc.php?rsd" /> <link rel="wlwmanifest" type="application/wlwmanifest+xml" href="https://nonint.com/wp-includes/wlwmanifest.xml" /> <meta name="generator" content="WordPress 5.9.10" /> <link rel="canonical" href="https://nonint.com/2023/07/01/techniques-for-debugging-neural-networks/" /> <link rel='shortlink' href='https://nonint.com/?p=318' /> <link rel="alternate" type="application/json+oembed" href="https://nonint.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fnonint.com%2F2023%2F07%2F01%2Ftechniques-for-debugging-neural-networks%2F" /> <script type="text/javascript"> (function(url){ if(/(?:Chrome\/26\.0\.1410\.63 Safari\/537\.31|WordfenceTestMonBot)/.test(navigator.userAgent)){ return; } var addEvent = function(evt, handler) { if (window.addEventListener) { document.addEventListener(evt, handler, false); } else if (window.attachEvent) { document.attachEvent('on' + evt, handler); } }; var removeEvent = function(evt, handler) { if (window.removeEventListener) { document.removeEventListener(evt, handler, false); } else if (window.detachEvent) { document.detachEvent('on' + evt, handler); } }; var evts = 'contextmenu dblclick drag dragend dragenter dragleave dragover dragstart drop keydown keypress keyup mousedown mousemove mouseout mouseover mouseup mousewheel scroll'.split(' '); var logHuman = function() { if (window.wfLogHumanRan) { return; } window.wfLogHumanRan = true; var wfscr = document.createElement('script'); wfscr.type = 'text/javascript'; wfscr.async = true; wfscr.src = url + '&r=' + Math.random(); (document.getElementsByTagName('head')[0]||document.getElementsByTagName('body')[0]).appendChild(wfscr); for (var i = 0; i < evts.length; i++) { removeEvent(evts[i], logHuman); } }; for (var i = 0; i < evts.length; i++) { addEvent(evts[i], logHuman); } })('//nonint.com/?wordfence_lh=1&hid=634091BBC1C3CF45B243DB7CD9670CC8'); </script> <style type="text/css"> .blogpost-button, .wp-block-search .wp-block-search__button, .comments-area p.form-submit input, .post-data-divider { background: ; } .super-menu, #smobile-menu, .primary-menu, .primary-menu ul li ul.children, .primary-menu ul li ul.sub-menu { background-color: ; } .main-navigation ul li a, .main-navigation ul li .sub-arrow, .super-menu .toggle-mobile-menu,.toggle-mobile-menu:before, .mobile-menu-active .smenu-hide { color: ; } #smobile-menu.show .main-navigation ul ul.children.active, #smobile-menu.show .main-navigation ul ul.sub-menu.active, #smobile-menu.show .main-navigation ul li, .smenu-hide.toggle-mobile-menu.menu-toggle, #smobile-menu.show .main-navigation ul li, .primary-menu ul li ul.children li, .primary-menu ul li ul.sub-menu li, .primary-menu .pmenu, .super-menu { border-color: ; border-bottom-color: ; } #secondary .widget h3, #secondary .widget h3 a, #secondary .widget h4, #secondary .widget h1, #secondary .widget h2, #secondary .widget h5, #secondary .widget h6 { color: ; } #secondary .widget a, #secondary a, #secondary .widget li a , #secondary span.sub-arrow{ color: ; } #secondary, #secondary .widget, #secondary .widget p, #secondary .widget li, .widget time.rpwe-time.published { color: ; } .swidgets-wrap .sidebar-title-border, #secondary .swidgets-wrap, #secondary .widget ul li, .featured-sidebar .search-field { border-color: ; } .site-info, .footer-column-three input.search-submit, .footer-column-three p, .footer-column-three li, .footer-column-three td, .footer-column-three th, .footer-column-three caption { color: ; } body, .site, .swidgets-wrap h3, .post-data-text { background: ; } .site-title a, .site-description { color: ; } .sheader { background: } </style> <style type="text/css"> body, .site, .swidgets-wrap h3, .post-data-text { background: ; } .site-title a, .site-description { color: ; } .sheader { background: } </style> <style type="text/css" id="wp-custom-css"> /* Posts Page */ #primary { width:100%; } /* Individual Post */ #primary-mono { width:100%; } pre.wp-block-code { border-left: 6px solid #3498DB; background-color: #000000; padding: 5px; } </style> </head> <body class="post-template-default single single-post postid-318 single-format-standard masthead-fixed"> <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-dark-grayscale"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0 0.49803921568627" /><feFuncG type="table" tableValues="0 0.49803921568627" /><feFuncB type="table" tableValues="0 0.49803921568627" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-grayscale"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0 1" /><feFuncG type="table" tableValues="0 1" /><feFuncB type="table" tableValues="0 1" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-purple-yellow"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0.54901960784314 0.98823529411765" /><feFuncG type="table" tableValues="0 1" /><feFuncB type="table" tableValues="0.71764705882353 0.25490196078431" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-blue-red"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0 1" /><feFuncG type="table" tableValues="0 0.27843137254902" /><feFuncB type="table" tableValues="0.5921568627451 0.27843137254902" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-midnight"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0 0" /><feFuncG type="table" tableValues="0 0.64705882352941" /><feFuncB type="table" tableValues="0 1" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-magenta-yellow"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0.78039215686275 1" /><feFuncG type="table" tableValues="0 0.94901960784314" /><feFuncB type="table" tableValues="0.35294117647059 0.47058823529412" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-purple-green"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0.65098039215686 0.40392156862745" /><feFuncG type="table" tableValues="0 1" /><feFuncB type="table" tableValues="0.44705882352941 0.4" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 0 0" width="0" height="0" focusable="false" role="none" style="visibility: hidden; position: absolute; left: -9999px; overflow: hidden;" ><defs><filter id="wp-duotone-blue-orange"><feColorMatrix color-interpolation-filters="sRGB" type="matrix" values=" .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 .299 .587 .114 0 0 " /><feComponentTransfer color-interpolation-filters="sRGB" ><feFuncR type="table" tableValues="0.098039215686275 1" /><feFuncG type="table" tableValues="0 0.66274509803922" /><feFuncB type="table" tableValues="0.84705882352941 0.41960784313725" /><feFuncA type="table" tableValues="1 1" /></feComponentTransfer><feComposite in2="SourceGraphic" operator="in" /></filter></defs></svg><a class="skip-link screen-reader-text" href="#content">Skip to content</a> <div id="page" class="site"> <header id="masthead" class="sheader site-header clearfix"> <div class="content-wrap"> <div class="site-branding">  <p class="site-title"><a href="https://nonint.com/" rel="home">Non_Interactive – Software & ML</a></p> </div> </div> <nav id="primary-site-navigation" class="primary-menu main-navigation clearfix"> <a href="#" id="pull" class="smenu-hide toggle-mobile-menu menu-toggle" aria-controls="secondary-menu" aria-expanded="false">Menu</a> <div class="content-wrap text-center"> <div class="center-main-menu"> <div id="primary-menu" class="pmenu"><ul> <li class="page_item page-item-10"><a href="https://nonint.com/contact/">Contact</a></li> <li class="page_item page-item-136"><a href="https://nonint.com/">Non_Int</a></li> <li class="page_item page-item-9"><a href="https://nonint.com/about/">What is Non-Interactive?</a></li> </ul></div> </div> </div> </nav> <div class="super-menu clearfix"> <div class="super-menu-inner"> <a href="#" id="pull" class="toggle-mobile-menu menu-toggle" aria-controls="secondary-menu" aria-expanded="false">Menu</a> </div> </div> <div id="mobile-menu-overlay"></div> </header> <div class="content-wrap"> <div class="header-widgets-wrapper"> </div> </div> <div id="content" class="site-content clearfix"> <div class="content-wrap"> <div id="primary" class="featured-content content-area"> <main id="main" class="site-main"> <article id="post-318" class="posts-entry fbox post-318 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title">Techniques for debugging neural networks</h1> <div class="entry-meta"> <div class="blog-data-wrapper"> <div class="post-data-divider"></div> <div class="post-data-positioning"> <div class="post-data-text"> <span class="posted-on">Posted on <a href="https://nonint.com/2023/07/01/techniques-for-debugging-neural-networks/" rel="bookmark"><time class="entry-date published" datetime="2023-07-01T22:52:17-06:00">July 1, 2023</time><time class="updated" datetime="2023-07-01T22:52:18-06:00">July 1, 2023</time></a></span><span class="byline"> by <span class="author vcard"><a class="url fn n" href="https://nonint.com/author/jbetker/">jbetker</a></span></span> </div> </div> </div> </div> </header> <div class="entry-content"> <p>In my last post, I briefly discussed the infuriating fact that a neural network, even when deeply flawed, will often “work” in the sense that it’ll do above-random at classification or a generative network might create things that may sometimes look plausibly from the dataset. </p> <p>Given an idea that you’re testing out that is performing poorly – how, then, do you tell the difference between a botched implementation and an idea that just isn’t good? I think this is one of the toughest questions I have to deal with on a daily basis as an ML engineer. It’s the difference between funneling an immense amount of work into an idea that doesn’t pan out (which happens often!) or calling it early to look at something else.</p> <p>I definitely don’t have all the answers, but I have gathered a few tricks over the last couple of years that I wanted to share:</p> <p>Know how to interpret your loss curves. Different classes of NNs will have different loss curve shapes, but rarely do tweaks to a NN result in a change to the shape – generally performance will increase or decrease in a stepwise fashion. New notches in the curves or progressively diverging performance is an interesting phenomenon you should consider digging into more. In particular – are you sure that you didn’t add more compute to your NN?</p> <p> For generative models, build a good eval and plot it regularly. This eval should actually sample an image/audio clip/etc from your generator and the eval should measure some aspect of your modality that you are not actively optimizing. My favorite approach here is to take a pre-trained classifier for a modality and use it to generate loss values between real/generated classes given a label. Evals that judge sampling results will often show performance differences that are not easily visible in the training loss.</p> <p>Plot grad and param norms. In runs that will likely go divergent before long, you’ll notice spikes in the grad norms start to occur. When training with FP16, these will occur at a regular intervals as the scales overflow, but will begin to occur more often as a NN becomes less stable. If you’re seeing an abnormal, or increasing level of grad spikes in your logs, it’s a good thing to consider stopping training runs and re-calibrating some of your hyperparameters (increased weight decay can reduce param and correspondingly grad norms, a lower learning rate can also help reduce both norms).</p> <p>Param norms are a good early warning sign for parameters which are overfitting some aspect of the dataset. These seem to ultimately be the cause of grad spikes and ultimately training divergence. Things to look for are param norms that appear to be growing without an upper bound. If you plot the norm for all parameters individually, you can sometimes catch this earlier, since sometimes this curve takes a long time to “warm” up for some parameters. The solution to exploding param norms is the same – increased weight decay or lower learning rates (or rethink some aspect of your architecture).</p> <p>Activation norms are a good thing to watch to judge the long term stability of your network. They should generally correlate pretty well parameter norms, but they also incorporate variance that can be attributed to the dataset. A good indication of a network that is becoming brittle and will soon fail is one with very high activation norms. This isn’t universally true, but in my experience it is a great way to compare the stability of two networks: If you have a baseline that works and a new network with higher activation norms, the latter will be more likely to diverge over the long term.</p> <p>Activation norms can also be used to isolate the specific causes of a failure. Generally only a small set of the norms will be divergent (for example, several of the projection weights for your MLP layers). Knowing where you are having numeric problems can help when trying to fix these types of problems – by adding normalization layers or L2 weight regularization, for example.</p> </div> </article> </main> </div> </div> </div> <footer id="colophon" class="site-footer clearfix"> <div class="content-wrap"> </div> <div class="site-info"> © 2024 Non_Interactive – Software & ML  <span class="footer-info-right"> | Powered by <a href="https://superbthemes.com/minimalistblogger/">Minimalist Blog</a> WordPress Theme </span>  </div> </div> </footer> </div>  <div id="smobile-menu" class="mobile-only"></div> <div id="mobile-menu-overlay"></div> <script type='text/javascript' src='https://nonint.com/wp-content/themes/minimalistblogger/js/navigation.js?ver=20170823' id='minimalistblogger-navigation-js'></script> <script type='text/javascript' src='https://nonint.com/wp-content/themes/minimalistblogger/js/skip-link-focus-fix.js?ver=20170823' id='minimalistblogger-skip-link-focus-fix-js'></script> <script type='text/javascript' src='https://nonint.com/wp-content/themes/minimalistblogger/js/script.js?ver=20160720' id='minimalistblogger-script-js'></script> <script type='text/javascript' src='https://nonint.com/wp-content/themes/minimalistblogger/js/jquery.flexslider.js?ver=20150423' id='flexslider-js'></script> <script type='text/javascript' src='https://nonint.com/wp-content/themes/minimalistblogger/js/accessibility.js?ver=20160720' id='minimalistblogger-accessibility-js'></script> </body> </html>