CINXE.COM
GitHub - wenwei202/terngrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow)
<!DOCTYPE html> <html lang="en" data-color-mode="auto" data-light-theme="light" data-dark-theme="dark" data-a11y-animated-images="system" data-a11y-link-underlines="true" > <head> <meta charset="utf-8"> <link rel="dns-prefetch" href="https://github.githubassets.com"> <link rel="dns-prefetch" href="https://avatars.githubusercontent.com"> <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com"> <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/"> <link rel="preconnect" href="https://github.githubassets.com" crossorigin> <link rel="preconnect" href="https://avatars.githubusercontent.com"> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/light-74231a1f3bbb.css" /><link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/dark-8a995f0bacd4.css" /><link data-color-theme="dark_dimmed" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_dimmed-f37fb7684b1f.css" /><link data-color-theme="dark_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_high_contrast-9ac301c3ebe5.css" /><link data-color-theme="dark_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_colorblind-cd826e8636dc.css" /><link data-color-theme="light_colorblind" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_colorblind-f91b0f603451.css" /><link data-color-theme="light_high_contrast" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_high_contrast-83beb16e0ecf.css" /><link data-color-theme="light_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/light_tritanopia-6e122dab64fc.css" /><link data-color-theme="dark_tritanopia" crossorigin="anonymous" media="all" rel="stylesheet" data-href="https://github.githubassets.com/assets/dark_tritanopia-18119e682df0.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-primitives-225433424a87.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-aaa714e5674d.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/global-7d4d2344e7ab.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/github-43ae85d4871b.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repository-4fce88777fa8.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/code-0210be90f4d3.css" /> <script type="application/json" id="client-env">{"locale":"en","featureFlags":["a11y_quote_reply_fix","copilot_immersive_issue_preview","copilot_new_references_ui","copilot_chat_repo_custom_instructions_preview","copilot_no_floating_button","copilot_topics_as_references","copilot_read_shared_conversation","copilot_duplicate_thread","copilot_buffered_streaming","dotcom_chat_client_side_skills","experimentation_azure_variant_endpoint","failbot_handle_non_errors","fgpat_form_ui_updates","geojson_azure_maps","ghost_pilot_confidence_truncation_25","ghost_pilot_confidence_truncation_40","github_models_o3_mini_streaming","hovercard_accessibility","insert_before_patch","issues_react_remove_placeholders","issues_react_blur_item_picker_on_close","marketing_pages_search_explore_provider","primer_react_css_modules_ga","react_data_router_pull_requests","remove_child_patch","sample_network_conn_type","swp_enterprise_contact_form","site_proxima_australia_update","viewscreen_sandbox","issues_react_create_milestone","issues_react_cache_fix_workaround","lifecycle_label_name_updates","copilot_task_oriented_assistive_prompts","issues_react_assignee_warning","issue_types_prevent_private_type_creation","turbo_app_id_restore"]}</script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/wp-runtime-7e63cc235734.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_oddbird_popover-polyfill_dist_popover_js-9da652f58479.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_arianotify-polyfill_ariaNotify-polyfill_js-node_modules_github_mi-3abb8f-46b9f4874d95.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_failbot_failbot_ts-75968cfb5298.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/environment-f04cb2a9fc8c.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_primer_behaviors_dist_esm_index_mjs-0dbb79f97f8f.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_selector-observer_dist_index_esm_js-f690fd9ae3d5.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_relative-time-element_dist_index_js-62d275b7ddd9.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_text-expander-element_dist_index_js-78748950cb0c.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_auto-complete-element_dist_index_js-node_modules_github_catalyst_-8e9f78-a90ac05d2469.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_filter-input-element_dist_index_js-node_modules_github_remote-inp-b5f1d7-a1760ffda83d.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_markdown-toolbar-element_dist_index_js-ceef33f593fa.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_file-attachment-element_dist_index_js-node_modules_primer_view-co-c44a69-8c52cf4cd0d3.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/github-elements-394f8eb34f19.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/element-registry-0bebfa1427c4.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_braintree_browser-detection_dist_browser-detection_js-node_modules_githu-2906d7-2a07a295af40.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_lit-html_lit-html_js-be8cb88f481b.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_morphdom_dist_morphdom-e-7c534c-a4a1922eb55f.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_turbo_dist_turbo_es2017-esm_js-a03ee12d659a.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-893f9f-b6294cf703b7.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_color-convert_index_js-e3180fe3bcb3.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_quote-selection_dist_index_js-node_modules_github_session-resume_-947061-e7a6c4a19f98.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_updatable-content_updatable-content_ts-2a55124d5c52.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_task-list_ts-app_assets_modules_github_sso_ts-ui_packages-900dde-768abe60b1f8.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_sticky-scroll-into-view_ts-3e000c5d31a9.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_ajax-error_ts-app_assets_modules_github_behaviors_include-87a4ae-8be71414579a.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_behaviors_commenting_edit_ts-app_assets_modules_github_behaviors_ht-83c235-e429cff6ceb1.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/behaviors-7ebb6421bf22.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_delegated-events_dist_index_js-node_modules_github_catalyst_lib_index_js-f6223d90c7ba.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/notifications-global-01e85cd1be94.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_virtualized-list_es_index_js-node_modules_github_template-parts_lib_index_js-94dc7a2157c1.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-70450e-4b93df70b903.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/app_assets_modules_github_ref-selector_ts-3e9d848bab5f.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/codespaces-c3bcacfe317c.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_filter-input-element_dist_index_js-node_modules_github_remote-inp-3eebbd-0763620ad7bf.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_decorators_js-node_modules_delegated-events_di-e161aa-9d41fb1b6c9e.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_file-attachment-element_dist_index_js-node_modules_github_remote--3c9c82-b71ef90fbdc7.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/repositories-7a0dbaa42c57.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_github_catalyst_lib_inde-dbbea9-26cce2010167.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/code-menu-1c0aedc134b1.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/primer-react-e05a7c4c5398.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/react-core-6a3ef8158713.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/react-lib-f1bca44e0926.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/octicons-react-cf2f2ab8dab4.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_emotion_is-prop-valid_dist_emotion-is-prop-valid_esm_js-node_modules_emo-62da9f-2df2f32ec596.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_mini-throttle_dist_index_js-node_modules_stacktrace-parser_dist_s-e7dcdd-9a233856b02c.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_oddbird_popover-polyfill_dist_popover-fn_js-55fea94174bf.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/notifications-subscriptions-menu-58a0c58bfee4.js"></script> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.e0c9f0687c56358ed85e.module.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.1bcff9205c241e99cff2.module.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.e0c9f0687c56358ed85e.module.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/notifications-subscriptions-menu.1bcff9205c241e99cff2.module.css" /> <title>GitHub - wenwei202/terngrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow)</title> <meta name="route-pattern" content="/:user_id/:repository" data-turbo-transient> <meta name="route-controller" content="files" data-turbo-transient> <meta name="route-action" content="disambiguate" data-turbo-transient> <meta name="current-catalog-service-hash" content="f3abb0cc802f3d7b95fc8762b94bdcb13bf39634c40c357301c4aa1d67a256fb"> <meta name="request-id" content="CDA6:18507A:C0B513:EC77C4:67E0C767" data-pjax-transient="true"/><meta name="html-safe-nonce" content="4adf7f6d77accbce4d26b44ae3084a960e65eb9d7b7a3f07d451e2a8c2aab207" data-pjax-transient="true"/><meta name="visitor-payload" content="eyJyZWZlcnJlciI6IiIsInJlcXVlc3RfaWQiOiJDREE2OjE4NTA3QTpDMEI1MTM6RUM3N0M0OjY3RTBDNzY3IiwidmlzaXRvcl9pZCI6IjUwMzc5ODM2NDE4ODQ5MzY3MSIsInJlZ2lvbl9lZGdlIjoic291dGhlYXN0YXNpYSIsInJlZ2lvbl9yZW5kZXIiOiJzb3V0aGVhc3Rhc2lhIn0=" data-pjax-transient="true"/><meta name="visitor-hmac" content="b998f7636ef0f5bed4a0d754643378c0e8a514ff8458233282f8e46c5a8fef63" data-pjax-transient="true"/> <meta name="hovercard-subject-tag" content="repository:91928571" data-turbo-transient> <meta name="github-keyboard-shortcuts" content="repository,copilot" data-turbo-transient="true" /> <meta name="selected-link" value="repo_source" data-turbo-transient> <link rel="assets" href="https://github.githubassets.com/"> <meta name="google-site-verification" content="Apib7-x98H0j5cPqHWwSMm6dNU4GmODRoqxLiDzdx9I"> <meta name="octolytics-url" content="https://collector.github.com/github/collect" /> <meta name="analytics-location" content="/<user-name>/<repo-name>" data-turbo-transient="true" /> <meta name="user-login" content=""> <meta name="viewport" content="width=device-width"> <meta name="description" content="Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) - wenwei202/terngrad"> <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub"> <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub"> <meta property="fb:app_id" content="1401488693436528"> <meta name="apple-itunes-app" content="app-id=1477376905, app-argument=https://github.com/wenwei202/terngrad" /> <meta name="twitter:image" content="https://opengraph.githubassets.com/522634a3465d8635845dca89be4dc79707cf333c1790252aae57392bbb1962ce/wenwei202/terngrad" /><meta name="twitter:site" content="@github" /><meta name="twitter:card" content="summary_large_image" /><meta name="twitter:title" content="GitHub - wenwei202/terngrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow)" /><meta name="twitter:description" content="Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) - wenwei202/terngrad" /> <meta property="og:image" content="https://opengraph.githubassets.com/522634a3465d8635845dca89be4dc79707cf333c1790252aae57392bbb1962ce/wenwei202/terngrad" /><meta property="og:image:alt" content="Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) - wenwei202/terngrad" /><meta property="og:image:width" content="1200" /><meta property="og:image:height" content="600" /><meta property="og:site_name" content="GitHub" /><meta property="og:type" content="object" /><meta property="og:title" content="GitHub - wenwei202/terngrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow)" /><meta property="og:url" content="https://github.com/wenwei202/terngrad" /><meta property="og:description" content="Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) - wenwei202/terngrad" /> <meta name="hostname" content="github.com"> <meta name="expected-hostname" content="github.com"> <meta http-equiv="x-pjax-version" content="052a71b7c02165fbf2715bdbfbab05187e4b16e53a581d8594996a90b2e7e0b4" data-turbo-track="reload"> <meta http-equiv="x-pjax-csp-version" content="77190eb53eb47fc30bd2fcc17a7eefa2dfd8505869fee9299ba911be3a40a9eb" data-turbo-track="reload"> <meta http-equiv="x-pjax-css-version" content="3c046d4b918199cbb4d255cca63226853d0617dbae6529663229394aa9b0742a" data-turbo-track="reload"> <meta http-equiv="x-pjax-js-version" content="1529793745024e41638e0c5e8b0d9e42797bb9de3789d89787088aa632e6de83" data-turbo-track="reload"> <meta name="turbo-cache-control" content="no-preview" data-turbo-transient=""> <meta data-hydrostats="publish"> <meta name="go-import" content="github.com/wenwei202/terngrad git https://github.com/wenwei202/terngrad.git"> <meta name="octolytics-dimension-user_id" content="12142066" /><meta name="octolytics-dimension-user_login" content="wenwei202" /><meta name="octolytics-dimension-repository_id" content="91928571" /><meta name="octolytics-dimension-repository_nwo" content="wenwei202/terngrad" /><meta name="octolytics-dimension-repository_public" content="true" /><meta name="octolytics-dimension-repository_is_fork" content="false" /><meta name="octolytics-dimension-repository_network_root_id" content="91928571" /><meta name="octolytics-dimension-repository_network_root_nwo" content="wenwei202/terngrad" /> <link rel="canonical" href="https://github.com/wenwei202/terngrad" data-turbo-transient> <meta name="turbo-body-classes" content="logged-out env-production page-responsive"> <meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats"> <meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors"> <meta name="release" content="36e3ad0da9839a7a57c910b6cf2f0d45ca56838f"> <link rel="mask-icon" href="https://github.githubassets.com/assets/pinned-octocat-093da3e6fa40.svg" color="#000000"> <link rel="alternate icon" class="js-site-favicon" type="image/png" href="https://github.githubassets.com/favicons/favicon.png"> <link rel="icon" class="js-site-favicon" type="image/svg+xml" href="https://github.githubassets.com/favicons/favicon.svg" data-base-href="https://github.githubassets.com/favicons/favicon"> <meta name="theme-color" content="#1e2327"> <meta name="color-scheme" content="light dark" /> <link rel="manifest" href="/manifest.json" crossOrigin="use-credentials"> </head> <body class="logged-out env-production page-responsive" style="word-wrap: break-word;"> <div data-turbo-body class="logged-out env-production page-responsive" style="word-wrap: break-word;"> <div class="position-relative header-wrapper js-header-wrapper "> <a href="#start-of-content" data-skip-target-assigned="false" class="px-2 py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content">Skip to content</a> <span data-view-component="true" class="progress-pjax-loader Progress position-fixed width-full"> <span style="width: 0%;" data-view-component="true" class="Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis"></span> </span> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_ui-commands_ui-commands_ts-8c874fb594e9.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/keyboard-shortcuts-dialog-33dfb803e078.js"></script> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.e0c9f0687c56358ed85e.module.css" /> <react-partial partial-name="keyboard-shortcuts-dialog" data-ssr="false" data-attempted-ssr="false" > <script type="application/json" data-target="react-partial.embeddedData">{"props":{"docsUrl":"https://docs.github.com/get-started/accessibility/keyboard-shortcuts"}}</script> <div data-target="react-partial.reactRoot"></div> </react-partial> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_github_remote-form_dist_index_js-node_modules_delegated-events_dist_inde-94fd67-4898d1bf4b51.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/sessions-730dca81d0a2.js"></script> <header class="HeaderMktg header-logged-out js-details-container js-header Details f4 py-3" role="banner" data-is-top="true" data-color-mode=light data-light-theme=light data-dark-theme=dark> <h2 class="sr-only">Navigation Menu</h2> <button type="button" class="HeaderMktg-backdrop d-lg-none border-0 position-fixed top-0 left-0 width-full height-full js-details-target" aria-label="Toggle navigation"> <span class="d-none">Toggle navigation</span> </button> <div class="d-flex flex-column flex-lg-row flex-items-center px-3 px-md-4 px-lg-5 height-full position-relative z-1"> <div class="d-flex flex-justify-between flex-items-center width-full width-lg-auto"> <div class="flex-1"> <button aria-label="Toggle navigation" aria-expanded="false" type="button" data-view-component="true" class="js-details-target js-nav-padding-recalculate js-header-menu-toggle Button--link Button--medium Button d-lg-none color-fg-inherit p-1"> <span class="Button-content"> <span class="Button-label"><div class="HeaderMenu-toggle-bar rounded my-1"></div> <div class="HeaderMenu-toggle-bar rounded my-1"></div> <div class="HeaderMenu-toggle-bar rounded my-1"></div></span> </span> </button> </div> <a class="mr-lg-3 color-fg-inherit flex-order-2 js-prevent-focus-on-mobile-nav" href="/" aria-label="Homepage" data-analytics-event="{"category":"Marketing nav","action":"click to go to homepage","label":"ref_page:Marketing;ref_cta:Logomark;ref_loc:Header"}"> <svg height="32" aria-hidden="true" viewBox="0 0 24 24" version="1.1" width="32" data-view-component="true" class="octicon octicon-mark-github"> <path d="M12 1C5.9225 1 1 5.9225 1 12C1 16.8675 4.14875 20.9787 8.52125 22.4362C9.07125 22.5325 9.2775 22.2025 9.2775 21.9137C9.2775 21.6525 9.26375 20.7862 9.26375 19.865C6.5 20.3737 5.785 19.1912 5.565 18.5725C5.44125 18.2562 4.905 17.28 4.4375 17.0187C4.0525 16.8125 3.5025 16.3037 4.42375 16.29C5.29 16.2762 5.90875 17.0875 6.115 17.4175C7.105 19.0812 8.68625 18.6137 9.31875 18.325C9.415 17.61 9.70375 17.1287 10.02 16.8537C7.5725 16.5787 5.015 15.63 5.015 11.4225C5.015 10.2262 5.44125 9.23625 6.1425 8.46625C6.0325 8.19125 5.6475 7.06375 6.2525 5.55125C6.2525 5.55125 7.17375 5.2625 9.2775 6.67875C10.1575 6.43125 11.0925 6.3075 12.0275 6.3075C12.9625 6.3075 13.8975 6.43125 14.7775 6.67875C16.8813 5.24875 17.8025 5.55125 17.8025 5.55125C18.4075 7.06375 18.0225 8.19125 17.9125 8.46625C18.6138 9.23625 19.04 10.2125 19.04 11.4225C19.04 15.6437 16.4688 16.5787 14.0213 16.8537C14.42 17.1975 14.7638 17.8575 14.7638 18.8887C14.7638 20.36 14.75 21.5425 14.75 21.9137C14.75 22.2025 14.9563 22.5462 15.5063 22.4362C19.8513 20.9787 23 16.8537 23 12C23 5.9225 18.0775 1 12 1Z"></path> </svg> </a> <div class="flex-1 flex-order-2 text-right"> <a href="/login?return_to=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad" class="HeaderMenu-link HeaderMenu-button d-inline-flex d-lg-none flex-order-1 f5 no-underline border color-border-default rounded-2 px-2 py-1 color-fg-inherit js-prevent-focus-on-mobile-nav" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="0bcb63cee95c25dc04e7b170e5a600cb6323db05015178d6f29ddee7d9f7e749" data-analytics-event="{"category":"Marketing nav","action":"click to Sign in","label":"ref_page:Marketing;ref_cta:Sign in;ref_loc:Header"}" > Sign in </a> </div> </div> <div class="HeaderMenu js-header-menu height-fit position-lg-relative d-lg-flex flex-column flex-auto top-0"> <div class="HeaderMenu-wrapper d-flex flex-column flex-self-start flex-lg-row flex-auto rounded rounded-lg-0"> <nav class="HeaderMenu-nav" aria-label="Global"> <ul class="d-lg-flex list-style-none"> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false"> Product <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1"> <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path> </svg> </button> <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 d-lg-flex flex-wrap dropdown-menu-wide"> <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0"> <ul class="list-style-none f5" > <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"github_copilot","context":"product","tag":"link","label":"github_copilot_link_product_navbar"}" href="https://github.com/features/copilot"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-copilot color-fg-subtle mr-3"> <path d="M23.922 16.992c-.861 1.495-5.859 5.023-11.922 5.023-6.063 0-11.061-3.528-11.922-5.023A.641.641 0 0 1 0 16.736v-2.869a.841.841 0 0 1 .053-.22c.372-.935 1.347-2.292 2.605-2.656.167-.429.414-1.055.644-1.517a10.195 10.195 0 0 1-.052-1.086c0-1.331.282-2.499 1.132-3.368.397-.406.89-.717 1.474-.952 1.399-1.136 3.392-2.093 6.122-2.093 2.731 0 4.767.957 6.166 2.093.584.235 1.077.546 1.474.952.85.869 1.132 2.037 1.132 3.368 0 .368-.014.733-.052 1.086.23.462.477 1.088.644 1.517 1.258.364 2.233 1.721 2.605 2.656a.832.832 0 0 1 .053.22v2.869a.641.641 0 0 1-.078.256ZM12.172 11h-.344a4.323 4.323 0 0 1-.355.508C10.703 12.455 9.555 13 7.965 13c-1.725 0-2.989-.359-3.782-1.259a2.005 2.005 0 0 1-.085-.104L4 11.741v6.585c1.435.779 4.514 2.179 8 2.179 3.486 0 6.565-1.4 8-2.179v-6.585l-.098-.104s-.033.045-.085.104c-.793.9-2.057 1.259-3.782 1.259-1.59 0-2.738-.545-3.508-1.492a4.323 4.323 0 0 1-.355-.508h-.016.016Zm.641-2.935c.136 1.057.403 1.913.878 2.497.442.544 1.134.938 2.344.938 1.573 0 2.292-.337 2.657-.751.384-.435.558-1.15.558-2.361 0-1.14-.243-1.847-.705-2.319-.477-.488-1.319-.862-2.824-1.025-1.487-.161-2.192.138-2.533.529-.269.307-.437.808-.438 1.578v.021c0 .265.021.562.063.893Zm-1.626 0c.042-.331.063-.628.063-.894v-.02c-.001-.77-.169-1.271-.438-1.578-.341-.391-1.046-.69-2.533-.529-1.505.163-2.347.537-2.824 1.025-.462.472-.705 1.179-.705 2.319 0 1.211.175 1.926.558 2.361.365.414 1.084.751 2.657.751 1.21 0 1.902-.394 2.344-.938.475-.584.742-1.44.878-2.497Z"></path><path d="M14.5 14.25a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Zm-5 0a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Z"></path> </svg> <div> <div class="color-fg-default h4">GitHub Copilot</div> Write better code with AI </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"security","context":"product","tag":"link","label":"security_link_product_navbar"}" href="https://github.com/features/security"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-shield-check color-fg-subtle mr-3"> <path d="M16.53 9.78a.75.75 0 0 0-1.06-1.06L11 13.19l-1.97-1.97a.75.75 0 0 0-1.06 1.06l2.5 2.5a.75.75 0 0 0 1.06 0l5-5Z"></path><path d="m12.54.637 8.25 2.675A1.75 1.75 0 0 1 22 4.976V10c0 6.19-3.771 10.704-9.401 12.83a1.704 1.704 0 0 1-1.198 0C5.77 20.705 2 16.19 2 10V4.976c0-.758.489-1.43 1.21-1.664L11.46.637a1.748 1.748 0 0 1 1.08 0Zm-.617 1.426-8.25 2.676a.249.249 0 0 0-.173.237V10c0 5.46 3.28 9.483 8.43 11.426a.199.199 0 0 0 .14 0C17.22 19.483 20.5 15.461 20.5 10V4.976a.25.25 0 0 0-.173-.237l-8.25-2.676a.253.253 0 0 0-.154 0Z"></path> </svg> <div> <div class="color-fg-default h4">Security</div> Find and fix vulnerabilities </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"actions","context":"product","tag":"link","label":"actions_link_product_navbar"}" href="https://github.com/features/actions"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-workflow color-fg-subtle mr-3"> <path d="M1 3a2 2 0 0 1 2-2h6.5a2 2 0 0 1 2 2v6.5a2 2 0 0 1-2 2H7v4.063C7 16.355 7.644 17 8.438 17H12.5v-2.5a2 2 0 0 1 2-2H21a2 2 0 0 1 2 2V21a2 2 0 0 1-2 2h-6.5a2 2 0 0 1-2-2v-2.5H8.437A2.939 2.939 0 0 1 5.5 15.562V11.5H3a2 2 0 0 1-2-2Zm2-.5a.5.5 0 0 0-.5.5v6.5a.5.5 0 0 0 .5.5h6.5a.5.5 0 0 0 .5-.5V3a.5.5 0 0 0-.5-.5ZM14.5 14a.5.5 0 0 0-.5.5V21a.5.5 0 0 0 .5.5H21a.5.5 0 0 0 .5-.5v-6.5a.5.5 0 0 0-.5-.5Z"></path> </svg> <div> <div class="color-fg-default h4">Actions</div> Automate any workflow </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"codespaces","context":"product","tag":"link","label":"codespaces_link_product_navbar"}" href="https://github.com/features/codespaces"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-codespaces color-fg-subtle mr-3"> <path d="M3.5 3.75C3.5 2.784 4.284 2 5.25 2h13.5c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 18.75 13H5.25a1.75 1.75 0 0 1-1.75-1.75Zm-2 12c0-.966.784-1.75 1.75-1.75h17.5c.966 0 1.75.784 1.75 1.75v4a1.75 1.75 0 0 1-1.75 1.75H3.25a1.75 1.75 0 0 1-1.75-1.75ZM5.25 3.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h13.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Zm-2 12a.25.25 0 0 0-.25.25v4c0 .138.112.25.25.25h17.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25Z"></path><path d="M10 17.75a.75.75 0 0 1 .75-.75h6.5a.75.75 0 0 1 0 1.5h-6.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path> </svg> <div> <div class="color-fg-default h4">Codespaces</div> Instant dev environments </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"issues","context":"product","tag":"link","label":"issues_link_product_navbar"}" href="https://github.com/features/issues"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-issue-opened color-fg-subtle mr-3"> <path d="M12 1c6.075 0 11 4.925 11 11s-4.925 11-11 11S1 18.075 1 12 5.925 1 12 1ZM2.5 12a9.5 9.5 0 0 0 9.5 9.5 9.5 9.5 0 0 0 9.5-9.5A9.5 9.5 0 0 0 12 2.5 9.5 9.5 0 0 0 2.5 12Zm9.5 2a2 2 0 1 1-.001-3.999A2 2 0 0 1 12 14Z"></path> </svg> <div> <div class="color-fg-default h4">Issues</div> Plan and track work </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"code_review","context":"product","tag":"link","label":"code_review_link_product_navbar"}" href="https://github.com/features/code-review"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-code-review color-fg-subtle mr-3"> <path d="M10.3 6.74a.75.75 0 0 1-.04 1.06l-2.908 2.7 2.908 2.7a.75.75 0 1 1-1.02 1.1l-3.5-3.25a.75.75 0 0 1 0-1.1l3.5-3.25a.75.75 0 0 1 1.06.04Zm3.44 1.06a.75.75 0 1 1 1.02-1.1l3.5 3.25a.75.75 0 0 1 0 1.1l-3.5 3.25a.75.75 0 1 1-1.02-1.1l2.908-2.7-2.908-2.7Z"></path><path d="M1.5 4.25c0-.966.784-1.75 1.75-1.75h17.5c.966 0 1.75.784 1.75 1.75v12.5a1.75 1.75 0 0 1-1.75 1.75h-9.69l-3.573 3.573A1.458 1.458 0 0 1 5 21.043V18.5H3.25a1.75 1.75 0 0 1-1.75-1.75ZM3.25 4a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h2.5a.75.75 0 0 1 .75.75v3.19l3.72-3.72a.749.749 0 0 1 .53-.22h10a.25.25 0 0 0 .25-.25V4.25a.25.25 0 0 0-.25-.25Z"></path> </svg> <div> <div class="color-fg-default h4">Code Review</div> Manage code changes </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"discussions","context":"product","tag":"link","label":"discussions_link_product_navbar"}" href="https://github.com/features/discussions"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-comment-discussion color-fg-subtle mr-3"> <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 14.25 14H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 15.543V14H1.75A1.75 1.75 0 0 1 0 12.25v-9.5C0 1.784.784 1 1.75 1ZM1.5 2.75v9.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-9.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Z"></path><path d="M22.5 8.75a.25.25 0 0 0-.25-.25h-3.5a.75.75 0 0 1 0-1.5h3.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 22.25 20H21v1.543a1.457 1.457 0 0 1-2.487 1.03L15.939 20H10.75A1.75 1.75 0 0 1 9 18.25v-1.465a.75.75 0 0 1 1.5 0v1.465c0 .138.112.25.25.25h5.5a.75.75 0 0 1 .53.22l2.72 2.72v-2.19a.75.75 0 0 1 .75-.75h2a.25.25 0 0 0 .25-.25v-9.5Z"></path> </svg> <div> <div class="color-fg-default h4">Discussions</div> Collaborate outside of code </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{"location":"navbar","action":"code_search","context":"product","tag":"link","label":"code_search_link_product_navbar"}" href="https://github.com/features/code-search"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-code-square color-fg-subtle mr-3"> <path d="M10.3 8.24a.75.75 0 0 1-.04 1.06L7.352 12l2.908 2.7a.75.75 0 1 1-1.02 1.1l-3.5-3.25a.75.75 0 0 1 0-1.1l3.5-3.25a.75.75 0 0 1 1.06.04Zm3.44 1.06a.75.75 0 1 1 1.02-1.1l3.5 3.25a.75.75 0 0 1 0 1.1l-3.5 3.25a.75.75 0 1 1-1.02-1.1l2.908-2.7-2.908-2.7Z"></path><path d="M2 3.75C2 2.784 2.784 2 3.75 2h16.5c.966 0 1.75.784 1.75 1.75v16.5A1.75 1.75 0 0 1 20.25 22H3.75A1.75 1.75 0 0 1 2 20.25Zm1.75-.25a.25.25 0 0 0-.25.25v16.5c0 .138.112.25.25.25h16.5a.25.25 0 0 0 .25-.25V3.75a.25.25 0 0 0-.25-.25Z"></path> </svg> <div> <div class="color-fg-default h4">Code Search</div> Find more, search less </div> </a></li> </ul> </div> </div> <div class="HeaderMenu-column px-lg-4"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 border-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="product-explore-heading">Explore</span> <ul class="list-style-none f5" aria-labelledby="product-explore-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"all_features","context":"product","tag":"link","label":"all_features_link_product_navbar"}" href="https://github.com/features"> All features </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"documentation","context":"product","tag":"link","label":"documentation_link_product_navbar"}" href="https://docs.github.com"> Documentation <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"github_skills","context":"product","tag":"link","label":"github_skills_link_product_navbar"}" href="https://skills.github.com"> GitHub Skills <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"blog","context":"product","tag":"link","label":"blog_link_product_navbar"}" href="https://github.blog"> Blog <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> </ul> </div> </div> </div> </li> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false"> Solutions <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1"> <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path> </svg> </button> <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 d-lg-flex flex-wrap dropdown-menu-wide"> <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 pb-lg-3 mb-3 mb-lg-0"> <span class="d-block h4 color-fg-default my-1" id="solutions-by-company-size-heading">By company size</span> <ul class="list-style-none f5" aria-labelledby="solutions-by-company-size-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"enterprises","context":"solutions","tag":"link","label":"enterprises_link_solutions_navbar"}" href="https://github.com/enterprise"> Enterprises </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"small_and_medium_teams","context":"solutions","tag":"link","label":"small_and_medium_teams_link_solutions_navbar"}" href="https://github.com/team"> Small and medium teams </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"startups","context":"solutions","tag":"link","label":"startups_link_solutions_navbar"}" href="https://github.com/enterprise/startups"> Startups </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"nonprofits","context":"solutions","tag":"link","label":"nonprofits_link_solutions_navbar"}" href="/solutions/industry/nonprofits"> Nonprofits </a></li> </ul> </div> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="solutions-by-use-case-heading">By use case</span> <ul class="list-style-none f5" aria-labelledby="solutions-by-use-case-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"devsecops","context":"solutions","tag":"link","label":"devsecops_link_solutions_navbar"}" href="/solutions/use-case/devsecops"> DevSecOps </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"devops","context":"solutions","tag":"link","label":"devops_link_solutions_navbar"}" href="/solutions/use-case/devops"> DevOps </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"ci_cd","context":"solutions","tag":"link","label":"ci_cd_link_solutions_navbar"}" href="/solutions/use-case/ci-cd"> CI/CD </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"view_all_use_cases","context":"solutions","tag":"link","label":"view_all_use_cases_link_solutions_navbar"}" href="/solutions/use-case"> View all use cases </a></li> </ul> </div> </div> <div class="HeaderMenu-column px-lg-4"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="solutions-by-industry-heading">By industry</span> <ul class="list-style-none f5" aria-labelledby="solutions-by-industry-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"healthcare","context":"solutions","tag":"link","label":"healthcare_link_solutions_navbar"}" href="/solutions/industry/healthcare"> Healthcare </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"financial_services","context":"solutions","tag":"link","label":"financial_services_link_solutions_navbar"}" href="/solutions/industry/financial-services"> Financial services </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"manufacturing","context":"solutions","tag":"link","label":"manufacturing_link_solutions_navbar"}" href="/solutions/industry/manufacturing"> Manufacturing </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"government","context":"solutions","tag":"link","label":"government_link_solutions_navbar"}" href="/solutions/industry/government"> Government </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"view_all_industries","context":"solutions","tag":"link","label":"view_all_industries_link_solutions_navbar"}" href="/solutions/industry"> View all industries </a></li> </ul> </div> </div> <div class="HeaderMenu-trailing-link rounded-bottom-2 flex-shrink-0 mt-lg-4 px-lg-4 py-4 py-lg-3 f5 text-semibold"> <a href="/solutions"> View all solutions <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-right HeaderMenu-trailing-link-icon"> <path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path> </svg> </a> </div> </div> </li> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false"> Resources <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1"> <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path> </svg> </button> <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 d-lg-flex flex-wrap dropdown-menu-wide"> <div class="HeaderMenu-column px-lg-4 border-lg-right mb-4 mb-lg-0 pr-lg-7"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="resources-topics-heading">Topics</span> <ul class="list-style-none f5" aria-labelledby="resources-topics-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"ai","context":"resources","tag":"link","label":"ai_link_resources_navbar"}" href="/resources/articles/ai"> AI </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"devops","context":"resources","tag":"link","label":"devops_link_resources_navbar"}" href="/resources/articles/devops"> DevOps </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"security","context":"resources","tag":"link","label":"security_link_resources_navbar"}" href="/resources/articles/security"> Security </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"software_development","context":"resources","tag":"link","label":"software_development_link_resources_navbar"}" href="/resources/articles/software-development"> Software Development </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"view_all","context":"resources","tag":"link","label":"view_all_link_resources_navbar"}" href="/resources/articles"> View all </a></li> </ul> </div> </div> <div class="HeaderMenu-column px-lg-4"> <div class="border-bottom pb-3 pb-lg-0 border-lg-bottom-0 border-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="resources-explore-heading">Explore</span> <ul class="list-style-none f5" aria-labelledby="resources-explore-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"learning_pathways","context":"resources","tag":"link","label":"learning_pathways_link_resources_navbar"}" href="https://resources.github.com/learn/pathways"> Learning Pathways <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"events_amp_webinars","context":"resources","tag":"link","label":"events_amp_webinars_link_resources_navbar"}" href="https://resources.github.com"> Events & Webinars <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"ebooks_amp_whitepapers","context":"resources","tag":"link","label":"ebooks_amp_whitepapers_link_resources_navbar"}" href="https://github.com/resources/whitepapers"> Ebooks & Whitepapers </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"customer_stories","context":"resources","tag":"link","label":"customer_stories_link_resources_navbar"}" href="https://github.com/customer-stories"> Customer Stories </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary Link--external" target="_blank" data-analytics-event="{"location":"navbar","action":"partners","context":"resources","tag":"link","label":"partners_link_resources_navbar"}" href="https://partner.github.com"> Partners <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle"> <path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path> </svg> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"executive_insights","context":"resources","tag":"link","label":"executive_insights_link_resources_navbar"}" href="https://github.com/solutions/executive-insights"> Executive Insights </a></li> </ul> </div> </div> </div> </li> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false"> Open Source <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1"> <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path> </svg> </button> <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 px-lg-4"> <div class="HeaderMenu-column"> <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3"> <ul class="list-style-none f5" > <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{"location":"navbar","action":"github_sponsors","context":"open_source","tag":"link","label":"github_sponsors_link_open_source_navbar"}" href="/sponsors"> <div> <div class="color-fg-default h4">GitHub Sponsors</div> Fund open source developers </div> </a></li> </ul> </div> <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3"> <ul class="list-style-none f5" > <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{"location":"navbar","action":"the_readme_project","context":"open_source","tag":"link","label":"the_readme_project_link_open_source_navbar"}" href="https://github.com/readme"> <div> <div class="color-fg-default h4">The ReadME Project</div> GitHub community articles </div> </a></li> </ul> </div> <div class="border-bottom pb-3 pb-lg-0 border-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="open-source-repositories-heading">Repositories</span> <ul class="list-style-none f5" aria-labelledby="open-source-repositories-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"topics","context":"open_source","tag":"link","label":"topics_link_open_source_navbar"}" href="https://github.com/topics"> Topics </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"trending","context":"open_source","tag":"link","label":"trending_link_open_source_navbar"}" href="https://github.com/trending"> Trending </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary" data-analytics-event="{"location":"navbar","action":"collections","context":"open_source","tag":"link","label":"collections_link_open_source_navbar"}" href="https://github.com/collections"> Collections </a></li> </ul> </div> </div> </div> </li> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <button type="button" class="HeaderMenu-link border-0 width-full width-lg-auto px-0 px-lg-2 py-lg-2 no-wrap d-flex flex-items-center flex-justify-between js-details-target" aria-expanded="false"> Enterprise <svg opacity="0.5" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-chevron-down HeaderMenu-icon ml-1"> <path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path> </svg> </button> <div class="HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 pt-2 pt-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 pb-2 pb-lg-4 px-lg-4"> <div class="HeaderMenu-column"> <div class="border-bottom pb-3 pb-lg-0 pb-lg-3 mb-3 mb-lg-0 mb-lg-3"> <ul class="list-style-none f5" > <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{"location":"navbar","action":"enterprise_platform","context":"enterprise","tag":"link","label":"enterprise_platform_link_enterprise_navbar"}" href="/enterprise"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-stack color-fg-subtle mr-3"> <path d="M11.063 1.456a1.749 1.749 0 0 1 1.874 0l8.383 5.316a1.751 1.751 0 0 1 0 2.956l-8.383 5.316a1.749 1.749 0 0 1-1.874 0L2.68 9.728a1.751 1.751 0 0 1 0-2.956Zm1.071 1.267a.25.25 0 0 0-.268 0L3.483 8.039a.25.25 0 0 0 0 .422l8.383 5.316a.25.25 0 0 0 .268 0l8.383-5.316a.25.25 0 0 0 0-.422Z"></path><path d="M1.867 12.324a.75.75 0 0 1 1.035-.232l8.964 5.685a.25.25 0 0 0 .268 0l8.964-5.685a.75.75 0 0 1 .804 1.267l-8.965 5.685a1.749 1.749 0 0 1-1.874 0l-8.965-5.685a.75.75 0 0 1-.231-1.035Z"></path><path d="M1.867 16.324a.75.75 0 0 1 1.035-.232l8.964 5.685a.25.25 0 0 0 .268 0l8.964-5.685a.75.75 0 0 1 .804 1.267l-8.965 5.685a1.749 1.749 0 0 1-1.874 0l-8.965-5.685a.75.75 0 0 1-.231-1.035Z"></path> </svg> <div> <div class="color-fg-default h4">Enterprise platform</div> AI-powered developer platform </div> </a></li> </ul> </div> <div class="border-bottom pb-3 pb-lg-0 border-bottom-0"> <span class="d-block h4 color-fg-default my-1" id="enterprise-available-add-ons-heading">Available add-ons</span> <ul class="list-style-none f5" aria-labelledby="enterprise-available-add-ons-heading"> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"advanced_security","context":"enterprise","tag":"link","label":"advanced_security_link_enterprise_navbar"}" href="https://github.com/enterprise/advanced-security"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-shield-check color-fg-subtle mr-3"> <path d="M16.53 9.78a.75.75 0 0 0-1.06-1.06L11 13.19l-1.97-1.97a.75.75 0 0 0-1.06 1.06l2.5 2.5a.75.75 0 0 0 1.06 0l5-5Z"></path><path d="m12.54.637 8.25 2.675A1.75 1.75 0 0 1 22 4.976V10c0 6.19-3.771 10.704-9.401 12.83a1.704 1.704 0 0 1-1.198 0C5.77 20.705 2 16.19 2 10V4.976c0-.758.489-1.43 1.21-1.664L11.46.637a1.748 1.748 0 0 1 1.08 0Zm-.617 1.426-8.25 2.676a.249.249 0 0 0-.173.237V10c0 5.46 3.28 9.483 8.43 11.426a.199.199 0 0 0 .14 0C17.22 19.483 20.5 15.461 20.5 10V4.976a.25.25 0 0 0-.173-.237l-8.25-2.676a.253.253 0 0 0-.154 0Z"></path> </svg> <div> <div class="color-fg-default h4">Advanced Security</div> Enterprise-grade security features </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description pb-lg-3" data-analytics-event="{"location":"navbar","action":"copilot_for_business","context":"enterprise","tag":"link","label":"copilot_for_business_link_enterprise_navbar"}" href="/features/copilot/copilot-business"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-copilot color-fg-subtle mr-3"> <path d="M23.922 16.992c-.861 1.495-5.859 5.023-11.922 5.023-6.063 0-11.061-3.528-11.922-5.023A.641.641 0 0 1 0 16.736v-2.869a.841.841 0 0 1 .053-.22c.372-.935 1.347-2.292 2.605-2.656.167-.429.414-1.055.644-1.517a10.195 10.195 0 0 1-.052-1.086c0-1.331.282-2.499 1.132-3.368.397-.406.89-.717 1.474-.952 1.399-1.136 3.392-2.093 6.122-2.093 2.731 0 4.767.957 6.166 2.093.584.235 1.077.546 1.474.952.85.869 1.132 2.037 1.132 3.368 0 .368-.014.733-.052 1.086.23.462.477 1.088.644 1.517 1.258.364 2.233 1.721 2.605 2.656a.832.832 0 0 1 .053.22v2.869a.641.641 0 0 1-.078.256ZM12.172 11h-.344a4.323 4.323 0 0 1-.355.508C10.703 12.455 9.555 13 7.965 13c-1.725 0-2.989-.359-3.782-1.259a2.005 2.005 0 0 1-.085-.104L4 11.741v6.585c1.435.779 4.514 2.179 8 2.179 3.486 0 6.565-1.4 8-2.179v-6.585l-.098-.104s-.033.045-.085.104c-.793.9-2.057 1.259-3.782 1.259-1.59 0-2.738-.545-3.508-1.492a4.323 4.323 0 0 1-.355-.508h-.016.016Zm.641-2.935c.136 1.057.403 1.913.878 2.497.442.544 1.134.938 2.344.938 1.573 0 2.292-.337 2.657-.751.384-.435.558-1.15.558-2.361 0-1.14-.243-1.847-.705-2.319-.477-.488-1.319-.862-2.824-1.025-1.487-.161-2.192.138-2.533.529-.269.307-.437.808-.438 1.578v.021c0 .265.021.562.063.893Zm-1.626 0c.042-.331.063-.628.063-.894v-.02c-.001-.77-.169-1.271-.438-1.578-.341-.391-1.046-.69-2.533-.529-1.505.163-2.347.537-2.824 1.025-.462.472-.705 1.179-.705 2.319 0 1.211.175 1.926.558 2.361.365.414 1.084.751 2.657.751 1.21 0 1.902-.394 2.344-.938.475-.584.742-1.44.878-2.497Z"></path><path d="M14.5 14.25a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Zm-5 0a1 1 0 0 1 1 1v2a1 1 0 0 1-2 0v-2a1 1 0 0 1 1-1Z"></path> </svg> <div> <div class="color-fg-default h4">Copilot for business</div> Enterprise-grade AI features </div> </a></li> <li> <a class="HeaderMenu-dropdown-link d-block no-underline position-relative py-2 Link--secondary d-flex flex-items-center Link--has-description" data-analytics-event="{"location":"navbar","action":"premium_support","context":"enterprise","tag":"link","label":"premium_support_link_enterprise_navbar"}" href="/premium-support"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-comment-discussion color-fg-subtle mr-3"> <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 14.25 14H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 15.543V14H1.75A1.75 1.75 0 0 1 0 12.25v-9.5C0 1.784.784 1 1.75 1ZM1.5 2.75v9.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-9.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Z"></path><path d="M22.5 8.75a.25.25 0 0 0-.25-.25h-3.5a.75.75 0 0 1 0-1.5h3.5c.966 0 1.75.784 1.75 1.75v9.5A1.75 1.75 0 0 1 22.25 20H21v1.543a1.457 1.457 0 0 1-2.487 1.03L15.939 20H10.75A1.75 1.75 0 0 1 9 18.25v-1.465a.75.75 0 0 1 1.5 0v1.465c0 .138.112.25.25.25h5.5a.75.75 0 0 1 .53.22l2.72 2.72v-2.19a.75.75 0 0 1 .75-.75h2a.25.25 0 0 0 .25-.25v-9.5Z"></path> </svg> <div> <div class="color-fg-default h4">Premium Support</div> Enterprise-grade 24/7 support </div> </a></li> </ul> </div> </div> </div> </li> <li class="HeaderMenu-item position-relative flex-wrap flex-justify-between flex-items-center d-block d-lg-flex flex-lg-nowrap flex-lg-items-center js-details-container js-header-menu-item"> <a class="HeaderMenu-link no-underline px-0 px-lg-2 py-3 py-lg-2 d-block d-lg-inline-block" data-analytics-event="{"location":"navbar","action":"pricing","context":"global","tag":"link","label":"pricing_link_global_navbar"}" href="https://github.com/pricing">Pricing</a> </li> </ul> </nav> <div class="d-flex flex-column flex-lg-row width-full flex-justify-end flex-lg-items-center text-center mt-3 mt-lg-0 text-lg-left ml-lg-3"> <qbsearch-input class="search-input" data-scope="repo:wenwei202/terngrad" data-custom-scopes-path="/search/custom_scopes" data-delete-custom-scopes-csrf="6RLSx2HIWHWaP7G3DXDq0hgsx4U_7tfx49dyb_y7vkptnc3_FSSfgKWBaqA7ds-c7MVOpiyuaF-1zRX-gQ-JMA" data-max-custom-scopes="10" data-header-redesign-enabled="false" data-initial-value="" data-blackbird-suggestions-path="/search/suggestions" data-jump-to-suggestions-path="/_graphql/GetSuggestedNavigationDestinations" data-current-repository="wenwei202/terngrad" data-current-org="" data-current-owner="wenwei202" data-logged-in="false" data-copilot-chat-enabled="false" data-nl-search-enabled="false" data-retain-scroll-position="true"> <div class="search-input-container search-with-dialog position-relative d-flex flex-row flex-items-center mr-4 rounded" data-action="click:qbsearch-input#searchInputContainerClicked" > <button type="button" class="header-search-button placeholder input-button form-control d-flex flex-1 flex-self-stretch flex-items-center no-wrap width-full py-0 pl-2 pr-0 text-left border-0 box-shadow-none" data-target="qbsearch-input.inputButton" aria-label="Search or jump to…" aria-haspopup="dialog" placeholder="Search or jump to..." data-hotkey=s,/ autocapitalize="off" data-analytics-event="{"location":"navbar","action":"searchbar","context":"global","tag":"input","label":"searchbar_input_global_navbar"}" data-action="click:qbsearch-input#handleExpand" > <div class="mr-2 color-fg-muted"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search"> <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path> </svg> </div> <span class="flex-1" data-target="qbsearch-input.inputButtonText">Search or jump to...</span> <div class="d-flex" data-target="qbsearch-input.hotkeyIndicator"> <svg xmlns="http://www.w3.org/2000/svg" width="22" height="20" aria-hidden="true" class="mr-1"><path fill="none" stroke="#979A9C" opacity=".4" d="M3.5.5h12c1.7 0 3 1.3 3 3v13c0 1.7-1.3 3-3 3h-12c-1.7 0-3-1.3-3-3v-13c0-1.7 1.3-3 3-3z"></path><path fill="#979A9C" d="M11.8 6L8 15.1h-.9L10.8 6h1z"></path></svg> </div> </button> <input type="hidden" name="type" class="js-site-search-type-field"> <div class="Overlay--hidden " data-modal-dialog-overlay> <modal-dialog data-action="close:qbsearch-input#handleClose cancel:qbsearch-input#handleClose" data-target="qbsearch-input.searchSuggestionsDialog" role="dialog" id="search-suggestions-dialog" aria-modal="true" aria-labelledby="search-suggestions-dialog-header" data-view-component="true" class="Overlay Overlay--width-large Overlay--height-auto"> <h1 id="search-suggestions-dialog-header" class="sr-only">Search code, repositories, users, issues, pull requests...</h1> <div class="Overlay-body Overlay-body--paddingNone"> <div data-view-component="true"> <div class="search-suggestions position-fixed width-full color-shadow-large border color-fg-default color-bg-default overflow-hidden d-flex flex-column query-builder-container" style="border-radius: 12px;" data-target="qbsearch-input.queryBuilderContainer" hidden > <!-- '"` --><!-- </textarea></xmp> --></option></form><form id="query-builder-test-form" action="" accept-charset="UTF-8" method="get"> <query-builder data-target="qbsearch-input.queryBuilder" id="query-builder-query-builder-test" data-filter-key=":" data-view-component="true" class="QueryBuilder search-query-builder"> <div class="FormControl FormControl--fullWidth"> <label id="query-builder-test-label" for="query-builder-test" class="FormControl-label sr-only"> Search </label> <div class="QueryBuilder-StyledInput width-fit " data-target="query-builder.styledInput" > <span id="query-builder-test-leadingvisual-wrap" class="FormControl-input-leadingVisualWrap QueryBuilder-leadingVisualWrap"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search FormControl-input-leadingVisual"> <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path> </svg> </span> <div data-target="query-builder.styledInputContainer" class="QueryBuilder-StyledInputContainer"> <div aria-hidden="true" class="QueryBuilder-StyledInputContent" data-target="query-builder.styledInputContent" ></div> <div class="QueryBuilder-InputWrapper"> <div aria-hidden="true" class="QueryBuilder-Sizer" data-target="query-builder.sizer"></div> <input id="query-builder-test" name="query-builder-test" value="" autocomplete="off" type="text" role="combobox" spellcheck="false" aria-expanded="false" aria-describedby="validation-8029e450-c223-4a95-beff-5713040a176c" data-target="query-builder.input" data-action=" input:query-builder#inputChange blur:query-builder#inputBlur keydown:query-builder#inputKeydown focus:query-builder#inputFocus " data-view-component="true" class="FormControl-input QueryBuilder-Input FormControl-medium" /> </div> </div> <span class="sr-only" id="query-builder-test-clear">Clear</span> <button role="button" id="query-builder-test-clear-button" aria-labelledby="query-builder-test-clear query-builder-test-label" data-target="query-builder.clearButton" data-action=" click:query-builder#clear focus:query-builder#clearButtonFocus blur:query-builder#clearButtonBlur " variant="small" hidden="hidden" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium mr-1 px-2 py-0 d-flex flex-items-center rounded-1 color-fg-muted"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x-circle-fill Button-visual"> <path d="M2.343 13.657A8 8 0 1 1 13.658 2.343 8 8 0 0 1 2.343 13.657ZM6.03 4.97a.751.751 0 0 0-1.042.018.751.751 0 0 0-.018 1.042L6.94 8 4.97 9.97a.749.749 0 0 0 .326 1.275.749.749 0 0 0 .734-.215L8 9.06l1.97 1.97a.749.749 0 0 0 1.275-.326.749.749 0 0 0-.215-.734L9.06 8l1.97-1.97a.749.749 0 0 0-.326-1.275.749.749 0 0 0-.734.215L8 6.94Z"></path> </svg> </button> </div> <template id="search-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-search"> <path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path> </svg> </template> <template id="code-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code"> <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path> </svg> </template> <template id="file-code-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-file-code"> <path d="M4 1.75C4 .784 4.784 0 5.75 0h5.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v8.586A1.75 1.75 0 0 1 14.25 15h-9a.75.75 0 0 1 0-1.5h9a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 10 4.25V1.5H5.75a.25.25 0 0 0-.25.25v2.5a.75.75 0 0 1-1.5 0Zm1.72 4.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734l1.47-1.47-1.47-1.47a.75.75 0 0 1 0-1.06ZM3.28 7.78 1.81 9.25l1.47 1.47a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Zm8.22-6.218V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path> </svg> </template> <template id="history-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-history"> <path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path> </svg> </template> <template id="repo-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo"> <path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path> </svg> </template> <template id="bookmark-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bookmark"> <path d="M3 2.75C3 1.784 3.784 1 4.75 1h6.5c.966 0 1.75.784 1.75 1.75v11.5a.75.75 0 0 1-1.227.579L8 11.722l-3.773 3.107A.751.751 0 0 1 3 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v9.91l3.023-2.489a.75.75 0 0 1 .954 0l3.023 2.49V2.75a.25.25 0 0 0-.25-.25Z"></path> </svg> </template> <template id="plus-circle-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-plus-circle"> <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm7.25-3.25v2.5h2.5a.75.75 0 0 1 0 1.5h-2.5v2.5a.75.75 0 0 1-1.5 0v-2.5h-2.5a.75.75 0 0 1 0-1.5h2.5v-2.5a.75.75 0 0 1 1.5 0Z"></path> </svg> </template> <template id="circle-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill"> <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path> </svg> </template> <template id="trash-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-trash"> <path d="M11 1.75V3h2.25a.75.75 0 0 1 0 1.5H2.75a.75.75 0 0 1 0-1.5H5V1.75C5 .784 5.784 0 6.75 0h2.5C10.216 0 11 .784 11 1.75ZM4.496 6.675l.66 6.6a.25.25 0 0 0 .249.225h5.19a.25.25 0 0 0 .249-.225l.66-6.6a.75.75 0 0 1 1.492.149l-.66 6.6A1.748 1.748 0 0 1 10.595 15h-5.19a1.75 1.75 0 0 1-1.741-1.575l-.66-6.6a.75.75 0 1 1 1.492-.15ZM6.5 1.75V3h3V1.75a.25.25 0 0 0-.25-.25h-2.5a.25.25 0 0 0-.25.25Z"></path> </svg> </template> <template id="team-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-people"> <path d="M2 5.5a3.5 3.5 0 1 1 5.898 2.549 5.508 5.508 0 0 1 3.034 4.084.75.75 0 1 1-1.482.235 4 4 0 0 0-7.9 0 .75.75 0 0 1-1.482-.236A5.507 5.507 0 0 1 3.102 8.05 3.493 3.493 0 0 1 2 5.5ZM11 4a3.001 3.001 0 0 1 2.22 5.018 5.01 5.01 0 0 1 2.56 3.012.749.749 0 0 1-.885.954.752.752 0 0 1-.549-.514 3.507 3.507 0 0 0-2.522-2.372.75.75 0 0 1-.574-.73v-.352a.75.75 0 0 1 .416-.672A1.5 1.5 0 0 0 11 5.5.75.75 0 0 1 11 4Zm-5.5-.5a2 2 0 1 0-.001 3.999A2 2 0 0 0 5.5 3.5Z"></path> </svg> </template> <template id="project-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-project"> <path d="M1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0ZM1.5 1.75v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25ZM11.75 3a.75.75 0 0 1 .75.75v7.5a.75.75 0 0 1-1.5 0v-7.5a.75.75 0 0 1 .75-.75Zm-8.25.75a.75.75 0 0 1 1.5 0v5.5a.75.75 0 0 1-1.5 0ZM8 3a.75.75 0 0 1 .75.75v3.5a.75.75 0 0 1-1.5 0v-3.5A.75.75 0 0 1 8 3Z"></path> </svg> </template> <template id="pencil-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pencil"> <path d="M11.013 1.427a1.75 1.75 0 0 1 2.474 0l1.086 1.086a1.75 1.75 0 0 1 0 2.474l-8.61 8.61c-.21.21-.47.364-.756.445l-3.251.93a.75.75 0 0 1-.927-.928l.929-3.25c.081-.286.235-.547.445-.758l8.61-8.61Zm.176 4.823L9.75 4.81l-6.286 6.287a.253.253 0 0 0-.064.108l-.558 1.953 1.953-.558a.253.253 0 0 0 .108-.064Zm1.238-3.763a.25.25 0 0 0-.354 0L10.811 3.75l1.439 1.44 1.263-1.263a.25.25 0 0 0 0-.354Z"></path> </svg> </template> <template id="copilot-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copilot"> <path d="M7.998 15.035c-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.201-.508-.254-1.084-.254-1.656 0-.87.128-1.769.693-2.484.579-.733 1.494-1.124 2.724-1.261 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095v1.872c0 .766-3.351 3.795-8.002 3.795Zm0-1.485c2.28 0 4.584-1.11 5.002-1.433V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-1.146 0-2.059-.327-2.71-.991A3.222 3.222 0 0 1 8 6.303a3.24 3.24 0 0 1-.544.743c-.65.664-1.563.991-2.71.991-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433ZM6.762 2.83c-.193-.206-.637-.413-1.682-.297-1.019.113-1.479.404-1.713.7-.247.312-.369.789-.369 1.554 0 .793.129 1.171.308 1.371.162.181.519.379 1.442.379.853 0 1.339-.235 1.638-.54.315-.322.527-.827.617-1.553.117-.935-.037-1.395-.241-1.614Zm4.155-.297c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Z"></path><path d="M6.25 9.037a.75.75 0 0 1 .75.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 .75-.75Zm4.25.75v1.501a.75.75 0 0 1-1.5 0V9.787a.75.75 0 0 1 1.5 0Z"></path> </svg> </template> <template id="copilot-error-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copilot-error"> <path d="M16 11.24c0 .112-.072.274-.21.467L13 9.688V7.862l-.023-.116c-.49.21-1.075.291-1.727.291-.198 0-.388-.009-.571-.029L6.833 5.226a4.01 4.01 0 0 0 .17-.782c.117-.935-.037-1.395-.241-1.614-.193-.206-.637-.413-1.682-.297-.683.076-1.115.231-1.395.415l-1.257-.91c.579-.564 1.413-.877 2.485-.996 1.206-.134 2.262.034 2.944.765.05.053.096.108.139.165.044-.057.094-.112.143-.165.682-.731 1.738-.899 2.944-.765 1.23.137 2.145.528 2.724 1.261.566.715.693 1.614.693 2.484 0 .572-.053 1.148-.254 1.656.066.228.098.429.126.612.012.076.024.148.037.218.924.385 1.522 1.471 1.591 2.095Zm-5.083-8.707c-1.044-.116-1.488.091-1.681.297-.204.219-.359.679-.242 1.614.091.726.303 1.231.618 1.553.299.305.784.54 1.638.54.922 0 1.28-.198 1.442-.379.179-.2.308-.578.308-1.371 0-.765-.123-1.242-.37-1.554-.233-.296-.693-.587-1.713-.7Zm2.511 11.074c-1.393.776-3.272 1.428-5.43 1.428-4.562 0-7.873-2.914-7.998-3.749V9.338c.085-.628.677-1.686 1.588-2.065.013-.07.024-.143.036-.218.029-.183.06-.384.126-.612-.18-.455-.241-.963-.252-1.475L.31 4.107A.747.747 0 0 1 0 3.509V3.49a.748.748 0 0 1 .625-.73c.156-.026.306.047.435.139l14.667 10.578a.592.592 0 0 1 .227.264.752.752 0 0 1 .046.249v.022a.75.75 0 0 1-1.19.596Zm-1.367-.991L5.635 7.964a5.128 5.128 0 0 1-.889.073c-.652 0-1.236-.081-1.727-.291l-.023.116v4.255c.419.323 2.722 1.433 5.002 1.433 1.539 0 3.089-.505 4.063-.934Z"></path> </svg> </template> <template id="workflow-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-workflow"> <path d="M0 1.75C0 .784.784 0 1.75 0h3.5C6.216 0 7 .784 7 1.75v3.5A1.75 1.75 0 0 1 5.25 7H4v4a1 1 0 0 0 1 1h4v-1.25C9 9.784 9.784 9 10.75 9h3.5c.966 0 1.75.784 1.75 1.75v3.5A1.75 1.75 0 0 1 14.25 16h-3.5A1.75 1.75 0 0 1 9 14.25v-.75H5A2.5 2.5 0 0 1 2.5 11V7h-.75A1.75 1.75 0 0 1 0 5.25Zm1.75-.25a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Zm9 9a.25.25 0 0 0-.25.25v3.5c0 .138.112.25.25.25h3.5a.25.25 0 0 0 .25-.25v-3.5a.25.25 0 0 0-.25-.25Z"></path> </svg> </template> <template id="book-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-book"> <path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path> </svg> </template> <template id="code-review-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code-review"> <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0 1 14.25 13H8.061l-2.574 2.573A1.458 1.458 0 0 1 3 14.543V13H1.75A1.75 1.75 0 0 1 0 11.25v-8.5C0 1.784.784 1 1.75 1ZM1.5 2.75v8.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h6.5a.25.25 0 0 0 .25-.25v-8.5a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Zm5.28 1.72a.75.75 0 0 1 0 1.06L5.31 7l1.47 1.47a.751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018l-2-2a.75.75 0 0 1 0-1.06l2-2a.75.75 0 0 1 1.06 0Zm2.44 0a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L10.69 7 9.22 5.53a.75.75 0 0 1 0-1.06Z"></path> </svg> </template> <template id="codespaces-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-codespaces"> <path d="M0 11.25c0-.966.784-1.75 1.75-1.75h12.5c.966 0 1.75.784 1.75 1.75v3A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm2-9.5C2 .784 2.784 0 3.75 0h8.5C13.216 0 14 .784 14 1.75v5a1.75 1.75 0 0 1-1.75 1.75h-8.5A1.75 1.75 0 0 1 2 6.75Zm1.75-.25a.25.25 0 0 0-.25.25v5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-5a.25.25 0 0 0-.25-.25Zm-2 9.5a.25.25 0 0 0-.25.25v3c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-3a.25.25 0 0 0-.25-.25Z"></path><path d="M7 12.75a.75.75 0 0 1 .75-.75h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1-.75-.75Zm-4 0a.75.75 0 0 1 .75-.75h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1-.75-.75Z"></path> </svg> </template> <template id="comment-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-comment"> <path d="M1 2.75C1 1.784 1.784 1 2.75 1h10.5c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 13.25 12H9.06l-2.573 2.573A1.458 1.458 0 0 1 4 13.543V12H2.75A1.75 1.75 0 0 1 1 10.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h2a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h4.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path> </svg> </template> <template id="comment-discussion-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-comment-discussion"> <path d="M1.75 1h8.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 10.25 10H7.061l-2.574 2.573A1.458 1.458 0 0 1 2 11.543V10h-.25A1.75 1.75 0 0 1 0 8.25v-5.5C0 1.784.784 1 1.75 1ZM1.5 2.75v5.5c0 .138.112.25.25.25h1a.75.75 0 0 1 .75.75v2.19l2.72-2.72a.749.749 0 0 1 .53-.22h3.5a.25.25 0 0 0 .25-.25v-5.5a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25Zm13 2a.25.25 0 0 0-.25-.25h-.5a.75.75 0 0 1 0-1.5h.5c.966 0 1.75.784 1.75 1.75v5.5A1.75 1.75 0 0 1 14.25 12H14v1.543a1.458 1.458 0 0 1-2.487 1.03L9.22 12.28a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l2.22 2.22v-2.19a.75.75 0 0 1 .75-.75h1a.25.25 0 0 0 .25-.25Z"></path> </svg> </template> <template id="organization-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-organization"> <path d="M1.75 16A1.75 1.75 0 0 1 0 14.25V1.75C0 .784.784 0 1.75 0h8.5C11.216 0 12 .784 12 1.75v12.5c0 .085-.006.168-.018.25h2.268a.25.25 0 0 0 .25-.25V8.285a.25.25 0 0 0-.111-.208l-1.055-.703a.749.749 0 1 1 .832-1.248l1.055.703c.487.325.779.871.779 1.456v5.965A1.75 1.75 0 0 1 14.25 16h-3.5a.766.766 0 0 1-.197-.026c-.099.017-.2.026-.303.026h-3a.75.75 0 0 1-.75-.75V14h-1v1.25a.75.75 0 0 1-.75.75Zm-.25-1.75c0 .138.112.25.25.25H4v-1.25a.75.75 0 0 1 .75-.75h2.5a.75.75 0 0 1 .75.75v1.25h2.25a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25ZM3.75 6h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5ZM3 3.75A.75.75 0 0 1 3.75 3h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 3.75Zm4 3A.75.75 0 0 1 7.75 6h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 7 6.75ZM7.75 3h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5ZM3 9.75A.75.75 0 0 1 3.75 9h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 9.75ZM7.75 9h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5Z"></path> </svg> </template> <template id="rocket-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-rocket"> <path d="M14.064 0h.186C15.216 0 16 .784 16 1.75v.186a8.752 8.752 0 0 1-2.564 6.186l-.458.459c-.314.314-.641.616-.979.904v3.207c0 .608-.315 1.172-.833 1.49l-2.774 1.707a.749.749 0 0 1-1.11-.418l-.954-3.102a1.214 1.214 0 0 1-.145-.125L3.754 9.816a1.218 1.218 0 0 1-.124-.145L.528 8.717a.749.749 0 0 1-.418-1.11l1.71-2.774A1.748 1.748 0 0 1 3.31 4h3.204c.288-.338.59-.665.904-.979l.459-.458A8.749 8.749 0 0 1 14.064 0ZM8.938 3.623h-.002l-.458.458c-.76.76-1.437 1.598-2.02 2.5l-1.5 2.317 2.143 2.143 2.317-1.5c.902-.583 1.74-1.26 2.499-2.02l.459-.458a7.25 7.25 0 0 0 2.123-5.127V1.75a.25.25 0 0 0-.25-.25h-.186a7.249 7.249 0 0 0-5.125 2.123ZM3.56 14.56c-.732.732-2.334 1.045-3.005 1.148a.234.234 0 0 1-.201-.064.234.234 0 0 1-.064-.201c.103-.671.416-2.273 1.15-3.003a1.502 1.502 0 1 1 2.12 2.12Zm6.94-3.935c-.088.06-.177.118-.266.175l-2.35 1.521.548 1.783 1.949-1.2a.25.25 0 0 0 .119-.213ZM3.678 8.116 5.2 5.766c.058-.09.117-.178.176-.266H3.309a.25.25 0 0 0-.213.119l-1.2 1.95ZM12 5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path> </svg> </template> <template id="shield-check-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield-check"> <path d="m8.533.133 5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667l5.25-1.68a1.748 1.748 0 0 1 1.066 0Zm-.61 1.429.001.001-5.25 1.68a.251.251 0 0 0-.174.237V7c0 1.36.275 2.666 1.057 3.859.784 1.194 2.121 2.342 4.366 3.298a.196.196 0 0 0 .154 0c2.245-.957 3.582-2.103 4.366-3.297C13.225 9.666 13.5 8.358 13.5 7V3.48a.25.25 0 0 0-.174-.238l-5.25-1.68a.25.25 0 0 0-.153 0ZM11.28 6.28l-3.5 3.5a.75.75 0 0 1-1.06 0l-1.5-1.5a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215l.97.97 2.97-2.97a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path> </svg> </template> <template id="heart-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-heart"> <path d="m8 14.25.345.666a.75.75 0 0 1-.69 0l-.008-.004-.018-.01a7.152 7.152 0 0 1-.31-.17 22.055 22.055 0 0 1-3.434-2.414C2.045 10.731 0 8.35 0 5.5 0 2.836 2.086 1 4.25 1 5.797 1 7.153 1.802 8 3.02 8.847 1.802 10.203 1 11.75 1 13.914 1 16 2.836 16 5.5c0 2.85-2.045 5.231-3.885 6.818a22.066 22.066 0 0 1-3.744 2.584l-.018.01-.006.003h-.002ZM4.25 2.5c-1.336 0-2.75 1.164-2.75 3 0 2.15 1.58 4.144 3.365 5.682A20.58 20.58 0 0 0 8 13.393a20.58 20.58 0 0 0 3.135-2.211C12.92 9.644 14.5 7.65 14.5 5.5c0-1.836-1.414-3-2.75-3-1.373 0-2.609.986-3.029 2.456a.749.749 0 0 1-1.442 0C6.859 3.486 5.623 2.5 4.25 2.5Z"></path> </svg> </template> <template id="server-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-server"> <path d="M1.75 1h12.5c.966 0 1.75.784 1.75 1.75v4c0 .372-.116.717-.314 1 .198.283.314.628.314 1v4a1.75 1.75 0 0 1-1.75 1.75H1.75A1.75 1.75 0 0 1 0 12.75v-4c0-.358.109-.707.314-1a1.739 1.739 0 0 1-.314-1v-4C0 1.784.784 1 1.75 1ZM1.5 2.75v4c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25Zm.25 5.75a.25.25 0 0 0-.25.25v4c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-4a.25.25 0 0 0-.25-.25ZM7 4.75A.75.75 0 0 1 7.75 4h4.5a.75.75 0 0 1 0 1.5h-4.5A.75.75 0 0 1 7 4.75ZM7.75 10h4.5a.75.75 0 0 1 0 1.5h-4.5a.75.75 0 0 1 0-1.5ZM3 4.75A.75.75 0 0 1 3.75 4h.5a.75.75 0 0 1 0 1.5h-.5A.75.75 0 0 1 3 4.75ZM3.75 10h.5a.75.75 0 0 1 0 1.5h-.5a.75.75 0 0 1 0-1.5Z"></path> </svg> </template> <template id="globe-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-globe"> <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM5.78 8.75a9.64 9.64 0 0 0 1.363 4.177c.255.426.542.832.857 1.215.245-.296.551-.705.857-1.215A9.64 9.64 0 0 0 10.22 8.75Zm4.44-1.5a9.64 9.64 0 0 0-1.363-4.177c-.307-.51-.612-.919-.857-1.215a9.927 9.927 0 0 0-.857 1.215A9.64 9.64 0 0 0 5.78 7.25Zm-5.944 1.5H1.543a6.507 6.507 0 0 0 4.666 5.5c-.123-.181-.24-.365-.352-.552-.715-1.192-1.437-2.874-1.581-4.948Zm-2.733-1.5h2.733c.144-2.074.866-3.756 1.58-4.948.12-.197.237-.381.353-.552a6.507 6.507 0 0 0-4.666 5.5Zm10.181 1.5c-.144 2.074-.866 3.756-1.58 4.948-.12.197-.237.381-.353.552a6.507 6.507 0 0 0 4.666-5.5Zm2.733-1.5a6.507 6.507 0 0 0-4.666-5.5c.123.181.24.365.353.552.714 1.192 1.436 2.874 1.58 4.948Z"></path> </svg> </template> <template id="issue-opened-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened"> <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path> </svg> </template> <template id="device-mobile-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-device-mobile"> <path d="M3.75 0h8.5C13.216 0 14 .784 14 1.75v12.5A1.75 1.75 0 0 1 12.25 16h-8.5A1.75 1.75 0 0 1 2 14.25V1.75C2 .784 2.784 0 3.75 0ZM3.5 1.75v12.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25h-8.5a.25.25 0 0 0-.25.25ZM8 13a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path> </svg> </template> <template id="package-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-package"> <path d="m8.878.392 5.25 3.045c.54.314.872.89.872 1.514v6.098a1.75 1.75 0 0 1-.872 1.514l-5.25 3.045a1.75 1.75 0 0 1-1.756 0l-5.25-3.045A1.75 1.75 0 0 1 1 11.049V4.951c0-.624.332-1.201.872-1.514L7.122.392a1.75 1.75 0 0 1 1.756 0ZM7.875 1.69l-4.63 2.685L8 7.133l4.755-2.758-4.63-2.685a.248.248 0 0 0-.25 0ZM2.5 5.677v5.372c0 .09.047.171.125.216l4.625 2.683V8.432Zm6.25 8.271 4.625-2.683a.25.25 0 0 0 .125-.216V5.677L8.75 8.432Z"></path> </svg> </template> <template id="credit-card-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-credit-card"> <path d="M10.75 9a.75.75 0 0 0 0 1.5h1.5a.75.75 0 0 0 0-1.5h-1.5Z"></path><path d="M0 3.75C0 2.784.784 2 1.75 2h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0 1 14.25 14H1.75A1.75 1.75 0 0 1 0 12.25ZM14.5 6.5h-13v5.75c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25Zm0-2.75a.25.25 0 0 0-.25-.25H1.75a.25.25 0 0 0-.25.25V5h13Z"></path> </svg> </template> <template id="play-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play"> <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path> </svg> </template> <template id="gift-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-gift"> <path d="M2 2.75A2.75 2.75 0 0 1 4.75 0c.983 0 1.873.42 2.57 1.232.268.318.497.668.68 1.042.183-.375.411-.725.68-1.044C9.376.42 10.266 0 11.25 0a2.75 2.75 0 0 1 2.45 4h.55c.966 0 1.75.784 1.75 1.75v2c0 .698-.409 1.301-1 1.582v4.918A1.75 1.75 0 0 1 13.25 16H2.75A1.75 1.75 0 0 1 1 14.25V9.332C.409 9.05 0 8.448 0 7.75v-2C0 4.784.784 4 1.75 4h.55c-.192-.375-.3-.8-.3-1.25ZM7.25 9.5H2.5v4.75c0 .138.112.25.25.25h4.5Zm1.5 0v5h4.5a.25.25 0 0 0 .25-.25V9.5Zm0-4V8h5.5a.25.25 0 0 0 .25-.25v-2a.25.25 0 0 0-.25-.25Zm-7 0a.25.25 0 0 0-.25.25v2c0 .138.112.25.25.25h5.5V5.5h-5.5Zm3-4a1.25 1.25 0 0 0 0 2.5h2.309c-.233-.818-.542-1.401-.878-1.793-.43-.502-.915-.707-1.431-.707ZM8.941 4h2.309a1.25 1.25 0 0 0 0-2.5c-.516 0-1 .205-1.43.707-.337.392-.646.975-.879 1.793Z"></path> </svg> </template> <template id="code-square-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code-square"> <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V1.75a.25.25 0 0 0-.25-.25Zm7.47 3.97a.75.75 0 0 1 1.06 0l2 2a.75.75 0 0 1 0 1.06l-2 2a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L10.69 8 9.22 6.53a.75.75 0 0 1 0-1.06ZM6.78 6.53 5.31 8l1.47 1.47a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-2-2a.75.75 0 0 1 0-1.06l2-2a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path> </svg> </template> <template id="device-desktop-icon"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-device-desktop"> <path d="M14.25 1c.966 0 1.75.784 1.75 1.75v7.5A1.75 1.75 0 0 1 14.25 12h-3.727c.099 1.041.52 1.872 1.292 2.757A.752.752 0 0 1 11.25 16h-6.5a.75.75 0 0 1-.565-1.243c.772-.885 1.192-1.716 1.292-2.757H1.75A1.75 1.75 0 0 1 0 10.25v-7.5C0 1.784.784 1 1.75 1ZM1.75 2.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25ZM9.018 12H6.982a5.72 5.72 0 0 1-.765 2.5h3.566a5.72 5.72 0 0 1-.765-2.5Z"></path> </svg> </template> <div class="position-relative"> <ul role="listbox" class="ActionListWrap QueryBuilder-ListWrap" aria-label="Suggestions" data-action=" combobox-commit:query-builder#comboboxCommit mousedown:query-builder#resultsMousedown " data-target="query-builder.resultsList" data-persist-list=false id="query-builder-test-results" ></ul> </div> <div class="FormControl-inlineValidation" id="validation-8029e450-c223-4a95-beff-5713040a176c" hidden="hidden"> <span class="FormControl-inlineValidation--visual"> <svg aria-hidden="true" height="12" viewBox="0 0 12 12" version="1.1" width="12" data-view-component="true" class="octicon octicon-alert-fill"> <path d="M4.855.708c.5-.896 1.79-.896 2.29 0l4.675 8.351a1.312 1.312 0 0 1-1.146 1.954H1.33A1.313 1.313 0 0 1 .183 9.058ZM7 7V3H5v4Zm-1 3a1 1 0 1 0 0-2 1 1 0 0 0 0 2Z"></path> </svg> </span> <span></span> </div> </div> <div data-target="query-builder.screenReaderFeedback" aria-live="polite" aria-atomic="true" class="sr-only"></div> </query-builder></form> <div class="d-flex flex-row color-fg-muted px-3 text-small color-bg-default search-feedback-prompt"> <a target="_blank" href="https://docs.github.com/search-github/github-code-search/understanding-github-code-search-syntax" data-view-component="true" class="Link color-fg-accent text-normal ml-2">Search syntax tips</a> <div class="d-flex flex-1"></div> </div> </div> </div> </div> </modal-dialog></div> </div> <div data-action="click:qbsearch-input#retract" class="dark-backdrop position-fixed" hidden data-target="qbsearch-input.darkBackdrop"></div> <div class="color-fg-default"> <dialog-helper> <dialog data-target="qbsearch-input.feedbackDialog" data-action="close:qbsearch-input#handleDialogClose cancel:qbsearch-input#handleDialogClose" id="feedback-dialog" aria-modal="true" aria-labelledby="feedback-dialog-title" aria-describedby="feedback-dialog-description" data-view-component="true" class="Overlay Overlay-whenNarrow Overlay--size-medium Overlay--motion-scaleFade Overlay--disableScroll"> <div data-view-component="true" class="Overlay-header"> <div class="Overlay-headerContentWrap"> <div class="Overlay-titleWrap"> <h1 class="Overlay-title " id="feedback-dialog-title"> Provide feedback </h1> </div> <div class="Overlay-actionWrap"> <button data-close-dialog-id="feedback-dialog" aria-label="Close" type="button" data-view-component="true" class="close-button Overlay-closeButton"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg></button> </div> </div> </div> <scrollable-region data-labelled-by="feedback-dialog-title"> <div data-view-component="true" class="Overlay-body"> <!-- '"` --><!-- </textarea></xmp> --></option></form><form id="code-search-feedback-form" data-turbo="false" action="/search/feedback" accept-charset="UTF-8" method="post"><input type="hidden" data-csrf="true" name="authenticity_token" value="5p2BYklx2AW+1AnBYjNQOmA85ZzDx8/Md54ifxFDShnLJ3qVashJ3PIgbwghfXTBJFxtTP29LCwiNpK1d45Gxw==" /> <p>We read every piece of feedback, and take your input very seriously.</p> <textarea name="feedback" class="form-control width-full mb-2" style="height: 120px" id="feedback"></textarea> <input name="include_email" id="include_email" aria-label="Include my email address so I can be contacted" class="form-control mr-2" type="checkbox"> <label for="include_email" style="font-weight: normal">Include my email address so I can be contacted</label> </form></div> </scrollable-region> <div data-view-component="true" class="Overlay-footer Overlay-footer--alignEnd"> <button data-close-dialog-id="feedback-dialog" type="button" data-view-component="true" class="btn"> Cancel </button> <button form="code-search-feedback-form" data-action="click:qbsearch-input#submitFeedback" type="submit" data-view-component="true" class="btn-primary btn"> Submit feedback </button> </div> </dialog></dialog-helper> <custom-scopes data-target="qbsearch-input.customScopesManager"> <dialog-helper> <dialog data-target="custom-scopes.customScopesModalDialog" data-action="close:qbsearch-input#handleDialogClose cancel:qbsearch-input#handleDialogClose" id="custom-scopes-dialog" aria-modal="true" aria-labelledby="custom-scopes-dialog-title" aria-describedby="custom-scopes-dialog-description" data-view-component="true" class="Overlay Overlay-whenNarrow Overlay--size-medium Overlay--motion-scaleFade Overlay--disableScroll"> <div data-view-component="true" class="Overlay-header Overlay-header--divided"> <div class="Overlay-headerContentWrap"> <div class="Overlay-titleWrap"> <h1 class="Overlay-title " id="custom-scopes-dialog-title"> Saved searches </h1> <h2 id="custom-scopes-dialog-description" class="Overlay-description">Use saved searches to filter your results more quickly</h2> </div> <div class="Overlay-actionWrap"> <button data-close-dialog-id="custom-scopes-dialog" aria-label="Close" type="button" data-view-component="true" class="close-button Overlay-closeButton"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg></button> </div> </div> </div> <scrollable-region data-labelled-by="custom-scopes-dialog-title"> <div data-view-component="true" class="Overlay-body"> <div data-target="custom-scopes.customScopesModalDialogFlash"></div> <div hidden class="create-custom-scope-form" data-target="custom-scopes.createCustomScopeForm"> <!-- '"` --><!-- </textarea></xmp> --></option></form><form id="custom-scopes-dialog-form" data-turbo="false" action="/search/custom_scopes" accept-charset="UTF-8" method="post"><input type="hidden" data-csrf="true" name="authenticity_token" value="89TcWW9xSmkRq0LtfZ1fUeMEXYcpTfrX22STyil+mniIp/SfdzfBfzQ7X3TMBhyWSe4UtxDwT2Lja9ntyjKfUw==" /> <div data-target="custom-scopes.customScopesModalDialogFlash"></div> <input type="hidden" id="custom_scope_id" name="custom_scope_id" data-target="custom-scopes.customScopesIdField"> <div class="form-group"> <label for="custom_scope_name">Name</label> <auto-check src="/search/custom_scopes/check_name" required only-validate-on-blur="false"> <input type="text" name="custom_scope_name" id="custom_scope_name" data-target="custom-scopes.customScopesNameField" class="form-control" autocomplete="off" placeholder="github-ruby" required maxlength="50"> <input type="hidden" data-csrf="true" value="g469iyoN6R1sxM7IE9X/FXnt7qqJAKWB7Tu5WnS0NXgiBGIJbRrCpR/wW4FnAfrmCDgQbiNcGvu+iIZ+ENpavg==" /> </auto-check> </div> <div class="form-group"> <label for="custom_scope_query">Query</label> <input type="text" name="custom_scope_query" id="custom_scope_query" data-target="custom-scopes.customScopesQueryField" class="form-control" autocomplete="off" placeholder="(repo:mona/a OR repo:mona/b) AND lang:python" required maxlength="500"> </div> <p class="text-small color-fg-muted"> To see all available qualifiers, see our <a class="Link--inTextBlock" href="https://docs.github.com/search-github/github-code-search/understanding-github-code-search-syntax">documentation</a>. </p> </form> </div> <div data-target="custom-scopes.manageCustomScopesForm"> <div data-target="custom-scopes.list"></div> </div> </div> </scrollable-region> <div data-view-component="true" class="Overlay-footer Overlay-footer--alignEnd Overlay-footer--divided"> <button data-action="click:custom-scopes#customScopesCancel" type="button" data-view-component="true" class="btn"> Cancel </button> <button form="custom-scopes-dialog-form" data-action="click:custom-scopes#customScopesSubmit" data-target="custom-scopes.customScopesSubmitButton" type="submit" data-view-component="true" class="btn-primary btn"> Create saved search </button> </div> </dialog></dialog-helper> </custom-scopes> </div> </qbsearch-input> <div class="position-relative HeaderMenu-link-wrap d-lg-inline-block"> <a href="/login?return_to=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad" class="HeaderMenu-link HeaderMenu-link--sign-in HeaderMenu-button flex-shrink-0 no-underline d-none d-lg-inline-flex border border-lg-0 rounded rounded-lg-0 px-2 py-1" style="margin-left: 12px;" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="0bcb63cee95c25dc04e7b170e5a600cb6323db05015178d6f29ddee7d9f7e749" data-analytics-event="{"category":"Marketing nav","action":"click to go to homepage","label":"ref_page:Marketing;ref_cta:Sign in;ref_loc:Header"}" > Sign in </a> </div> <a href="/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E&source=header-repo&source_repo=wenwei202%2Fterngrad" class="HeaderMenu-link HeaderMenu-link--sign-up HeaderMenu-button flex-shrink-0 d-flex d-lg-inline-flex no-underline border color-border-default rounded px-2 py-1" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"site header menu","repository_id":null,"auth_type":"SIGN_UP","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="0bcb63cee95c25dc04e7b170e5a600cb6323db05015178d6f29ddee7d9f7e749" data-analytics-event="{"category":"Sign up","action":"click to sign up for account","label":"ref_page:/<user-name>/<repo-name>;ref_cta:Sign up;ref_loc:header logged out"}" > Sign up </a> <button type="button" class="sr-only js-header-menu-focus-trap d-block d-lg-none">Reseting focus</button> </div> </div> </div> </div> </header> <div hidden="hidden" data-view-component="true" class="js-stale-session-flash stale-session-flash flash flash-warn flash-full"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert"> <path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path> </svg> <span class="js-stale-session-flash-signed-in" hidden>You signed in with another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span> <span class="js-stale-session-flash-signed-out" hidden>You signed out in another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span> <span class="js-stale-session-flash-switched" hidden>You switched accounts on another tab or window. <a class="Link--inTextBlock" href="">Reload</a> to refresh your session.</span> <button id="icon-button-cacddd3e-8af2-43ba-bbcb-778ad1904a54" aria-labelledby="tooltip-76de6fba-8c42-4fa3-be03-6d393a81a0b9" type="button" data-view-component="true" class="Button Button--iconOnly Button--invisible Button--medium flash-close js-flash-close"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x Button-visual"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg> </button><tool-tip id="tooltip-76de6fba-8c42-4fa3-be03-6d393a81a0b9" for="icon-button-cacddd3e-8af2-43ba-bbcb-778ad1904a54" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Dismiss alert</tool-tip> </div> </div> <div id="start-of-content" class="show-on-focus"></div> <div id="js-flash-container" class="flash-container" data-turbo-replace> <template class="js-flash-template"> <div class="flash flash-full {{ className }}"> <div > <button autofocus class="flash-close js-flash-close" type="button" aria-label="Dismiss this message"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg> </button> <div aria-atomic="true" role="alert" class="js-flash-alert"> <div>{{ message }}</div> </div> </div> </div> </template> </div> <div class="application-main " data-commit-hovercards-enabled data-discussion-hovercards-enabled data-issue-and-pr-hovercards-enabled data-project-hovercards-enabled > <div itemscope itemtype="http://schema.org/SoftwareSourceCode" class=""> <main id="js-repo-pjax-container" > <div id="repository-container-header" class="pt-3 hide-full-screen" style="background-color: var(--page-header-bgColor, var(--color-page-header-bg));" data-turbo-replace> <div class="d-flex flex-nowrap flex-justify-end mb-3 px-3 px-lg-5" style="gap: 1rem;"> <div class="flex-auto min-width-0 width-fit"> <div class=" d-flex flex-wrap flex-items-center wb-break-word f3 text-normal"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo color-fg-muted mr-2"> <path d="M2 2.5A2.5 2.5 0 0 1 4.5 0h8.75a.75.75 0 0 1 .75.75v12.5a.75.75 0 0 1-.75.75h-2.5a.75.75 0 0 1 0-1.5h1.75v-2h-8a1 1 0 0 0-.714 1.7.75.75 0 1 1-1.072 1.05A2.495 2.495 0 0 1 2 11.5Zm10.5-1h-8a1 1 0 0 0-1 1v6.708A2.486 2.486 0 0 1 4.5 9h8ZM5 12.25a.25.25 0 0 1 .25-.25h3.5a.25.25 0 0 1 .25.25v3.25a.25.25 0 0 1-.4.2l-1.45-1.087a.249.249 0 0 0-.3 0L5.4 15.7a.25.25 0 0 1-.4-.2Z"></path> </svg> <span class="author flex-self-stretch" itemprop="author"> <a class="url fn" rel="author" data-hovercard-type="user" data-hovercard-url="/users/wenwei202/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="/wenwei202"> wenwei202 </a> </span> <span class="mx-1 flex-self-stretch color-fg-muted">/</span> <strong itemprop="name" class="mr-2 flex-self-stretch"> <a data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" href="/wenwei202/terngrad">terngrad</a> </strong> <span></span><span class="Label Label--secondary v-align-middle mr-1">Public</span> </div> </div> <div id="repository-details-container" class="flex-shrink-0" data-turbo-replace style="max-width: 70%;"> <ul class="pagehead-actions flex-shrink-0 d-none d-md-inline" style="padding: 2px 0;"> <li> <a href="/login?return_to=%2Fwenwei202%2Fterngrad" rel="nofollow" id="repository-details-watch-button" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"notification subscription menu watch","repository_id":null,"auth_type":"LOG_IN","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="14e774fd3833335077e0a7e8526394492f933088e3ae12f7275a00e9baadce90" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2"> <path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path> </svg>Notifications </a> <tool-tip id="tooltip-bc4faf6c-960e-458f-ba61-fe492e61a53e" for="repository-details-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute">You must be signed in to change notification settings</tool-tip> </li> <li> <a icon="repo-forked" id="fork-button" href="/login?return_to=%2Fwenwei202%2Fterngrad" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"repo details fork button","repository_id":91928571,"auth_type":"LOG_IN","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="e34ebca4c65f84a31865347af291a524506ce3c3600a6c15e7ca46b8f2e3e557" data-view-component="true" class="btn-sm btn"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2"> <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path> </svg>Fork <span id="repo-network-counter" data-pjax-replace="true" data-turbo-replace="true" title="48" data-view-component="true" class="Counter">48</span> </a> </li> <li> <div data-view-component="true" class="BtnGroup d-flex"> <a href="/login?return_to=%2Fwenwei202%2Fterngrad" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"star button","repository_id":91928571,"auth_type":"LOG_IN","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="6060d026bb489795ef894d06ff293f89f3e45527bcbd76f70b4489558142b849" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2"> <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path> </svg><span data-view-component="true" class="d-inline"> Star </span> <span id="repo-stars-counter-star" aria-label="182 users starred this repository" data-singular-suffix="user starred this repository" data-plural-suffix="users starred this repository" data-turbo-replace="true" title="182" data-view-component="true" class="Counter js-social-count">182</span> </a></div> </li> </ul> </div> </div> <div id="responsive-meta-container" data-turbo-replace> <div class="d-block d-md-none mb-2 px-3 px-md-4 px-lg-5"> <p class="f4 mb-3 "> Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) </p> <h3 class="sr-only">License</h3> <div class="mb-2"> <a href="/wenwei202/terngrad/blob/master/LICENSE" class="Link--muted" data-analytics-event="{"category":"Repository Overview","action":"click","label":"location:sidebar;file:license"}" > <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-law mr-2"> <path d="M8.75.75V2h.985c.304 0 .603.08.867.231l1.29.736c.038.022.08.033.124.033h2.234a.75.75 0 0 1 0 1.5h-.427l2.111 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.006.005-.01.01-.045.04c-.21.176-.441.327-.686.45C14.556 10.78 13.88 11 13 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L12.178 4.5h-.162c-.305 0-.604-.079-.868-.231l-1.29-.736a.245.245 0 0 0-.124-.033H8.75V13h2.5a.75.75 0 0 1 0 1.5h-6.5a.75.75 0 0 1 0-1.5h2.5V3.5h-.984a.245.245 0 0 0-.124.033l-1.289.737c-.265.15-.564.23-.869.23h-.162l2.112 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.016.015-.045.04c-.21.176-.441.327-.686.45C4.556 10.78 3.88 11 3 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L2.178 4.5H1.75a.75.75 0 0 1 0-1.5h2.234a.249.249 0 0 0 .125-.033l1.288-.737c.265-.15.564-.23.869-.23h.984V.75a.75.75 0 0 1 1.5 0Zm2.945 8.477c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L13 6.327Zm-10 0c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L3 6.327Z"></path> </svg> Apache-2.0 license </a> </div> <div class="mb-3"> <a class="Link--secondary no-underline mr-3" href="/wenwei202/terngrad/stargazers"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star mr-1"> <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path> </svg> <span class="text-bold">182</span> stars </a> <a class="Link--secondary no-underline mr-3" href="/wenwei202/terngrad/forks"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-1"> <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path> </svg> <span class="text-bold">48</span> forks </a> <a class="Link--secondary no-underline mr-3 d-inline-block" href="/wenwei202/terngrad/branches"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-branch mr-1"> <path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path> </svg> <span>Branches</span> </a> <a class="Link--secondary no-underline d-inline-block" href="/wenwei202/terngrad/tags"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-tag mr-1"> <path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path> </svg> <span>Tags</span> </a> <a class="Link--secondary no-underline d-inline-block" href="/wenwei202/terngrad/activity"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pulse mr-1"> <path d="M6 2c.306 0 .582.187.696.471L10 10.731l1.304-3.26A.751.751 0 0 1 12 7h3.25a.75.75 0 0 1 0 1.5h-2.742l-1.812 4.528a.751.751 0 0 1-1.392 0L6 4.77 4.696 8.03A.75.75 0 0 1 4 8.5H.75a.75.75 0 0 1 0-1.5h2.742l1.812-4.529A.751.751 0 0 1 6 2Z"></path> </svg> <span>Activity</span> </a> </div> <div class="d-flex flex-wrap gap-2"> <div class="flex-1"> <div data-view-component="true" class="BtnGroup d-flex"> <a href="/login?return_to=%2Fwenwei202%2Fterngrad" rel="nofollow" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"star button","repository_id":91928571,"auth_type":"LOG_IN","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="6060d026bb489795ef894d06ff293f89f3e45527bcbd76f70b4489558142b849" aria-label="You must be signed in to star a repository" data-view-component="true" class="tooltipped tooltipped-sw btn-sm btn btn-block"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star v-align-text-bottom d-inline-block mr-2"> <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path> </svg><span data-view-component="true" class="d-inline"> Star </span> </a></div> </div> <div class="flex-1"> <a href="/login?return_to=%2Fwenwei202%2Fterngrad" rel="nofollow" id="files-overview-watch-button" data-hydro-click="{"event_type":"authentication.click","payload":{"location_in_page":"notification subscription menu watch","repository_id":null,"auth_type":"LOG_IN","originating_url":"https://github.com/wenwei202/terngrad","user_id":null}}" data-hydro-click-hmac="14e774fd3833335077e0a7e8526394492f933088e3ae12f7275a00e9baadce90" aria-label="You must be signed in to change notification settings" data-view-component="true" class="btn-sm btn btn-block"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-bell mr-2"> <path d="M8 16a2 2 0 0 0 1.985-1.75c.017-.137-.097-.25-.235-.25h-3.5c-.138 0-.252.113-.235.25A2 2 0 0 0 8 16ZM3 5a5 5 0 0 1 10 0v2.947c0 .05.015.098.042.139l1.703 2.555A1.519 1.519 0 0 1 13.482 13H2.518a1.516 1.516 0 0 1-1.263-2.36l1.703-2.554A.255.255 0 0 0 3 7.947Zm5-3.5A3.5 3.5 0 0 0 4.5 5v2.947c0 .346-.102.683-.294.97l-1.703 2.556a.017.017 0 0 0-.003.01l.001.006c0 .002.002.004.004.006l.006.004.007.001h10.964l.007-.001.006-.004.004-.006.001-.007a.017.017 0 0 0-.003-.01l-1.703-2.554a1.745 1.745 0 0 1-.294-.97V5A3.5 3.5 0 0 0 8 1.5Z"></path> </svg>Notifications </a> <tool-tip id="tooltip-ee705dc5-688c-4d8c-90da-836611aa51bf" for="files-overview-watch-button" popover="manual" data-direction="s" data-type="description" data-view-component="true" class="sr-only position-absolute">You must be signed in to change notification settings</tool-tip> </div> <span> </span> </div> </div> </div> <nav data-pjax="#js-repo-pjax-container" aria-label="Repository" data-view-component="true" class="js-repo-nav js-sidenav-container-pjax js-responsive-underlinenav overflow-hidden UnderlineNav px-3 px-md-4 px-lg-5"> <ul data-view-component="true" class="UnderlineNav-body list-style-none"> <li data-view-component="true" class="d-inline-flex"> <a id="code-tab" href="/wenwei202/terngrad" data-tab-item="i0code-tab" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches repo_packages repo_deployments repo_attestations /wenwei202/terngrad" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g c" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Code","target":"UNDERLINE_NAV.TAB"}" aria-current="page" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item selected"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code UnderlineNav-octicon d-none d-sm-inline"> <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path> </svg> <span data-content="Code">Code</span> <span id="code-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="issues-tab" href="/wenwei202/terngrad/issues" data-tab-item="i1issues-tab" data-selected-links="repo_issues repo_labels repo_milestones /wenwei202/terngrad/issues" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g i" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Issues","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened UnderlineNav-octicon d-none d-sm-inline"> <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path> </svg> <span data-content="Issues">Issues</span> <span id="issues-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="6" data-view-component="true" class="Counter">6</span> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="pull-requests-tab" href="/wenwei202/terngrad/pulls" data-tab-item="i2pull-requests-tab" data-selected-links="repo_pulls checks /wenwei202/terngrad/pulls" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g p" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Pull requests","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request UnderlineNav-octicon d-none d-sm-inline"> <path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path> </svg> <span data-content="Pull requests">Pull requests</span> <span id="pull-requests-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="actions-tab" href="/wenwei202/terngrad/actions" data-tab-item="i3actions-tab" data-selected-links="repo_actions /wenwei202/terngrad/actions" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g a" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Actions","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play UnderlineNav-octicon d-none d-sm-inline"> <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path> </svg> <span data-content="Actions">Actions</span> <span id="actions-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="projects-tab" href="/wenwei202/terngrad/projects" data-tab-item="i4projects-tab" data-selected-links="repo_projects new_repo_project repo_project /wenwei202/terngrad/projects" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g b" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Projects","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table UnderlineNav-octicon d-none d-sm-inline"> <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path> </svg> <span data-content="Projects">Projects</span> <span id="projects-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="0" hidden="hidden" data-view-component="true" class="Counter">0</span> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="security-tab" href="/wenwei202/terngrad/security" data-tab-item="i5security-tab" data-selected-links="security overview alerts policy token_scanning code_scanning /wenwei202/terngrad/security" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-hotkey="g s" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Security","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield UnderlineNav-octicon d-none d-sm-inline"> <path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path> </svg> <span data-content="Security">Security</span> <include-fragment src="/wenwei202/terngrad/security/overall-count" accept="text/fragment+html"></include-fragment> </a></li> <li data-view-component="true" class="d-inline-flex"> <a id="insights-tab" href="/wenwei202/terngrad/pulse" data-tab-item="i6insights-tab" data-selected-links="repo_graphs repo_contributors dependency_graph dependabot_updates pulse people community /wenwei202/terngrad/pulse" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame" data-analytics-event="{"category":"Underline navbar","action":"Click tab","label":"Insights","target":"UNDERLINE_NAV.TAB"}" data-view-component="true" class="UnderlineNav-item no-wrap js-responsive-underlinenav-item js-selected-navigation-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph UnderlineNav-octicon d-none d-sm-inline"> <path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path> </svg> <span data-content="Insights">Insights</span> <span id="insights-repo-tab-count" data-pjax-replace="" data-turbo-replace="" title="Not available" data-view-component="true" class="Counter"></span> </a></li> </ul> <div style="visibility:hidden;" data-view-component="true" class="UnderlineNav-actions js-responsive-underlinenav-overflow position-absolute pr-3 pr-md-4 pr-lg-5 right-0"> <action-menu data-select-variant="none" data-view-component="true"> <focus-group direction="vertical" mnemonics retain> <button id="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-button" popovertarget="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-overlay" aria-controls="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-list" aria-haspopup="true" aria-labelledby="tooltip-e4834a5d-ec1b-43a8-ba98-56ea8b749ae8" type="button" data-view-component="true" class="Button Button--iconOnly Button--secondary Button--medium UnderlineNav-item"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-kebab-horizontal Button-visual"> <path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path> </svg> </button><tool-tip id="tooltip-e4834a5d-ec1b-43a8-ba98-56ea8b749ae8" for="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-button" popover="manual" data-direction="s" data-type="label" data-view-component="true" class="sr-only position-absolute">Additional navigation options</tool-tip> <anchored-position data-target="action-menu.overlay" id="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-overlay" anchor="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-button" align="start" side="outside-bottom" anchor-offset="normal" popover="auto" data-view-component="true"> <div data-view-component="true" class="Overlay Overlay--size-auto"> <div data-view-component="true" class="Overlay-body Overlay-body--paddingNone"> <action-list> <div data-view-component="true"> <ul aria-labelledby="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-button" id="action-menu-5b64d764-6715-4754-8eec-c285e391cb08-list" role="menu" data-view-component="true" class="ActionListWrap--inset ActionListWrap"> <li hidden="hidden" data-menu-item="i0code-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-c217deff-0832-4b09-980e-5b1c5f1f91b9" href="/wenwei202/terngrad" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-code"> <path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Code </span> </a> </li> <li hidden="hidden" data-menu-item="i1issues-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-3fc553f7-c367-49f7-b025-a3125712849c" href="/wenwei202/terngrad/issues" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-issue-opened"> <path d="M8 9.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path><path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Issues </span> </a> </li> <li hidden="hidden" data-menu-item="i2pull-requests-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-b988ad93-a4f4-4d93-8a09-c50e585c421e" href="/wenwei202/terngrad/pulls" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-git-pull-request"> <path d="M1.5 3.25a2.25 2.25 0 1 1 3 2.122v5.256a2.251 2.251 0 1 1-1.5 0V5.372A2.25 2.25 0 0 1 1.5 3.25Zm5.677-.177L9.573.677A.25.25 0 0 1 10 .854V2.5h1A2.5 2.5 0 0 1 13.5 5v5.628a2.251 2.251 0 1 1-1.5 0V5a1 1 0 0 0-1-1h-1v1.646a.25.25 0 0 1-.427.177L7.177 3.427a.25.25 0 0 1 0-.354ZM3.75 2.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm0 9.5a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Zm8.25.75a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Pull requests </span> </a> </li> <li hidden="hidden" data-menu-item="i3actions-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-808dbb09-8baa-4538-a15a-6b6601fd4987" href="/wenwei202/terngrad/actions" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-play"> <path d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm4.879-2.773 4.264 2.559a.25.25 0 0 1 0 .428l-4.264 2.559A.25.25 0 0 1 6 10.559V5.442a.25.25 0 0 1 .379-.215Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Actions </span> </a> </li> <li hidden="hidden" data-menu-item="i4projects-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-0921d33d-c880-40da-a07a-374d85d7f3a2" href="/wenwei202/terngrad/projects" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-table"> <path d="M0 1.75C0 .784.784 0 1.75 0h12.5C15.216 0 16 .784 16 1.75v12.5A1.75 1.75 0 0 1 14.25 16H1.75A1.75 1.75 0 0 1 0 14.25ZM6.5 6.5v8h7.75a.25.25 0 0 0 .25-.25V6.5Zm8-1.5V1.75a.25.25 0 0 0-.25-.25H6.5V5Zm-13 1.5v7.75c0 .138.112.25.25.25H5v-8ZM5 5V1.5H1.75a.25.25 0 0 0-.25.25V5Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Projects </span> </a> </li> <li hidden="hidden" data-menu-item="i5security-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-32b4f4cc-5779-4b18-8129-406827779055" href="/wenwei202/terngrad/security" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-shield"> <path d="M7.467.133a1.748 1.748 0 0 1 1.066 0l5.25 1.68A1.75 1.75 0 0 1 15 3.48V7c0 1.566-.32 3.182-1.303 4.682-.983 1.498-2.585 2.813-5.032 3.855a1.697 1.697 0 0 1-1.33 0c-2.447-1.042-4.049-2.357-5.032-3.855C1.32 10.182 1 8.566 1 7V3.48a1.75 1.75 0 0 1 1.217-1.667Zm.61 1.429a.25.25 0 0 0-.153 0l-5.25 1.68a.25.25 0 0 0-.174.238V7c0 1.358.275 2.666 1.057 3.86.784 1.194 2.121 2.34 4.366 3.297a.196.196 0 0 0 .154 0c2.245-.956 3.582-2.104 4.366-3.298C13.225 9.666 13.5 8.36 13.5 7V3.48a.251.251 0 0 0-.174-.237l-5.25-1.68ZM8.75 4.75v3a.75.75 0 0 1-1.5 0v-3a.75.75 0 0 1 1.5 0ZM9 10.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Security </span> </a> </li> <li hidden="hidden" data-menu-item="i6insights-tab" data-targets="action-list.items" role="none" data-view-component="true" class="ActionListItem"> <a tabindex="-1" id="item-5241e986-13ad-49b0-94e4-b413c1c39e7c" href="/wenwei202/terngrad/pulse" role="menuitem" data-view-component="true" class="ActionListContent ActionListContent--visual16"> <span class="ActionListItem-visual ActionListItem-visual--leading"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-graph"> <path d="M1.5 1.75V13.5h13.75a.75.75 0 0 1 0 1.5H.75a.75.75 0 0 1-.75-.75V1.75a.75.75 0 0 1 1.5 0Zm14.28 2.53-5.25 5.25a.75.75 0 0 1-1.06 0L7 7.06 4.28 9.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.25-3.25a.75.75 0 0 1 1.06 0L10 7.94l4.72-4.72a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042Z"></path> </svg> </span> <span data-view-component="true" class="ActionListItem-label"> Insights </span> </a> </li> </ul> </div></action-list> </div> </div></anchored-position> </focus-group> </action-menu></div> </nav> </div> <turbo-frame id="repo-content-turbo-frame" target="_top" data-turbo-action="advance" class=""> <div id="repo-content-pjax-container" class="repository-content " > <h1 class='sr-only'>wenwei202/terngrad</h1> <div class="clearfix container-xl px-md-4 px-lg-5 px-3"> <div> <div style="max-width: 100%" data-view-component="true" class="Layout Layout--flowRow-until-md react-repos-overview-margin Layout--sidebarPosition-end Layout--sidebarPosition-flowRow-end"> <div data-view-component="true" class="Layout-main"> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_dompurify_dist_purify_es_mjs-dd1d3ea6a436.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/vendors-node_modules_tanstack_query-core_build_modern_queryObserver_js-node_modules_tanstack_-defd52-843b41414e0e.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_aria-live_aria-live_ts-ui_packages_promise-with-resolvers-polyfill_promise-with-r-17c672-34345cb18aac.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_paths_index_ts-e019c54eb886.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_ref-selector_RefSelector_tsx-7496afc3784d.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_commit-attribution_index_ts-ui_packages_commit-checks-status_index_ts-ui_packages-7094d4-15017f02e61c.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_code-view-shared_hooks_shortcuts_ts-ui_packages_code-view-shared_utilities_styles-0dc246-f8753c5db08d.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/ui_packages_code-view-shared_hooks_use-canonical-object_ts-ui_packages_code-view-shared_hooks-a83ec0-5ee2b562b57f.js"></script> <script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/repos-overview-ca785c0ab4fa.js"></script> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/primer-react.e0c9f0687c56358ed85e.module.css" /> <link crossorigin="anonymous" media="all" rel="stylesheet" href="https://github.githubassets.com/assets/repos-overview.0ee7cac3ab511a65d9f9.module.css" /> <react-partial partial-name="repos-overview" data-ssr="true" data-attempted-ssr="true" > <script type="application/json" data-target="react-partial.embeddedData">{"props":{"initialPayload":{"allShortcutsEnabled":false,"path":"/","repo":{"id":91928571,"defaultBranch":"master","name":"terngrad","ownerLogin":"wenwei202","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2017-05-21T02:34:51.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/12142066?v=4","public":true,"private":false,"isOrgOwned":false},"currentUser":null,"refInfo":{"name":"master","listCacheKey":"v0:1506565179.0","canEdit":false,"refType":"branch","currentOid":"ec4f75e9a3a1e1c4b2e6494d830fbdfdd2e03ddc"},"tree":{"items":[{"name":"slim","path":"slim","contentType":"directory"},{"name":"terngrad","path":"terngrad","contentType":"directory"},{"name":".gitignore","path":".gitignore","contentType":"file"},{"name":"LICENSE","path":"LICENSE","contentType":"file"},{"name":"NIPS17-TernGrad-slides-v3.pdf","path":"NIPS17-TernGrad-slides-v3.pdf","contentType":"file"},{"name":"Poster_Wen_NIPS2017.pdf","path":"Poster_Wen_NIPS2017.pdf","contentType":"file"},{"name":"README.md","path":"README.md","contentType":"file"}],"templateDirectorySuggestionUrl":null,"readme":null,"totalCount":7,"showBranchInfobar":false},"fileTree":null,"fileTreeProcessingTime":null,"foldersToFetch":[],"treeExpanded":false,"symbolsExpanded":false,"isOverview":true,"overview":{"banners":{"shouldRecommendReadme":false,"isPersonalRepo":false,"showUseActionBanner":false,"actionSlug":null,"actionId":null,"showProtectBranchBanner":false,"publishBannersInfo":{"dismissActionNoticePath":"/settings/dismiss-notice/publish_action_from_repo","releasePath":"/wenwei202/terngrad/releases/new?marketplace=true","showPublishActionBanner":false},"interactionLimitBanner":null,"showInvitationBanner":false,"inviterName":null,"actionsMigrationBannerInfo":{"releaseTags":[],"showImmutableActionsMigrationBanner":false,"initialMigrationStatus":null}},"codeButton":{"contactPath":"/contact","isEnterprise":false,"local":{"protocolInfo":{"httpAvailable":true,"sshAvailable":null,"httpUrl":"https://github.com/wenwei202/terngrad.git","showCloneWarning":null,"sshUrl":null,"sshCertificatesRequired":null,"sshCertificatesAvailable":null,"ghCliUrl":"gh repo clone wenwei202/terngrad","defaultProtocol":"http","newSshKeyUrl":"/settings/ssh/new","setProtocolPath":"/users/set_protocol"},"platformInfo":{"cloneUrl":"https://desktop.github.com","showVisualStudioCloneButton":false,"visualStudioCloneUrl":"https://windows.github.com","showXcodeCloneButton":false,"xcodeCloneUrl":"xcode://clone?repo=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad","zipballUrl":"/wenwei202/terngrad/archive/refs/heads/master.zip"}},"newCodespacePath":"/codespaces/new?hide_repo_select=true\u0026repo=91928571"},"popovers":{"rename":null,"renamedParentRepo":null},"commitCount":"280","overviewFiles":[{"displayName":"README.md","repoName":"terngrad","refName":"master","path":"README.md","preferredFileType":"readme","tabName":"README","richText":"\u003carticle class=\"markdown-body entry-content container-lg\" itemprop=\"text\"\u003e\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eIntroduction\u003c/h1\u003e\u003ca id=\"user-content-introduction\" class=\"anchor\" aria-label=\"Permalink: Introduction\" href=\"#introduction\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis repo is the TensorFlow code for our oral paper in NIPS 2017 (\u003ca href=\"https://papers.nips.cc/paper/6749-terngrad-ternary-gradients-to-reduce-communication-in-distributed-deep-learning.pdf\" rel=\"nofollow\"\u003eTernGrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning\u003c/a\u003e).\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eThe implementations of quantization and de-quantization operators are now available in \u003ca href=\"https://github.com/pytorch/pytorch/blob/master/caffe2/operators/fused_rowwise_random_quantization_ops.cc\"\u003eCaffe2/Pytorch 1.0\u003c/a\u003e (\u003ca href=\"https://github.com/pytorch/pytorch/blob/master/caffe2/python/operator_test/rand_quantization_op_test.py\"\u003ean example\u003c/a\u003e). TernGrad is landed into Facebook AI platforms in production to overcome communication bottleneck in large-scale training.\u003c/strong\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.youtube.com/watch?v=WWWQXTb_69c\u0026amp;feature=youtu.be\u0026amp;t=20s\" rel=\"nofollow\"\u003evideo\u003c/a\u003e, \u003ca href=\"/wenwei202/terngrad/blob/master/Poster_Wen_NIPS2017.pdf\"\u003eposter\u003c/a\u003e, \u003ca href=\"/wenwei202/terngrad/blob/master/NIPS17-TernGrad-slides-v3.pdf\"\u003eslides\u003c/a\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis is a modified copy of TensorFlow \u003ca href=\"https://github.com/tensorflow/models/tree/master/inception\"\u003einception\u003c/a\u003e (with original contributions kept).\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cem\u003eIn this workspace, \u003ccode\u003einception\u003c/code\u003e refers to all types of neural networks in a general way.\u003c/em\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cem\u003eNote that there is name abuse because of history reasons. All \"bingrad/binary gradient/binarizing\" in code comments, help info and filenames essentially refers to \"terngrad/ternary gradient/ternarizing\". Will update them, but the code is correct and is exactly for terngrad\u003c/em\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cem\u003eThe code is for algorithm evaluation only, i.e., -1,0,1 are still represented in floating precision. We are hacking TensorFlow/Caffe2 to support encoder/decoder of ternary gradients.\u003c/em\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eFor the interests in accelerating the inference of DNNs (\u003ca href=\"http://papers.nips.cc/paper/6504-learning-structured-sparsity-in-deep-neural-networks.pdf\" rel=\"nofollow\"\u003eNIPS 2016\u003c/a\u003e, \u003ca href=\"https://arxiv.org/abs/1703.09746\" rel=\"nofollow\"\u003eICCV 2017\u003c/a\u003e and \u003ca href=\"https://arxiv.org/abs/1709.05027\" rel=\"nofollow\"\u003eICLR 2018\u003c/a\u003e) using sparsity and low rank approximation, the code links are \u003ca href=\"https://github.com/wenwei202/caffe\"\u003ehere\u003c/a\u003e and \u003ca href=\"https://github.com/wenwei202/iss-rnns\"\u003ethere\u003c/a\u003e.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDependencies\u003c/h1\u003e\u003ca id=\"user-content-dependencies\" class=\"anchor\" aria-label=\"Permalink: Dependencies\" href=\"#dependencies\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eTested stable dependencies:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003epython 2.7 (Anaconda)\u003c/li\u003e\n\u003cli\u003eTensorflow v1.0.0 and v1.3.0\u003c/li\u003e\n\u003cli\u003ecudnn 5.1.5\u003c/li\u003e\n\u003cli\u003ebazel release 0.4.4\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003ePending to test by python 3.6.1 (Anaconda) and Tensorflow 1.1.0 (installed from python wheel). Known issues (mainly because of update to python 3):\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003euse \u003ccode\u003epickle\u003c/code\u003e instead of \u003ccode\u003ecPickle\u003c/code\u003e python package\u003c/li\u003e\n\u003cli\u003euse \u003ccode\u003erange\u003c/code\u003e instead of \u003ccode\u003exrange\u003c/code\u003e\u003c/li\u003e\n\u003cli\u003euse \u003ccode\u003edict.items\u003c/code\u003e instead of \u003ccode\u003edict.iteritems\u003c/code\u003e\u003c/li\u003e\n\u003cli\u003e\u003ccode\u003eTypeError: 'RGB' has type str, but expected one of: bytes\u003c/code\u003e: use \u003ccode\u003eb'RGB'\u003c/code\u003e instead of \u003ccode\u003e'RGB'\u003c/code\u003e\u003c/li\u003e\n\u003cli\u003e...\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBuild all\u003c/h1\u003e\u003ca id=\"user-content-build-all\" class=\"anchor\" aria-label=\"Permalink: Build all\" href=\"#build-all\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/terngrad\n./build_all.sh\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/terngrad\n./build_all.sh\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDataset preparation\u003c/h1\u003e\u003ca id=\"user-content-dataset-preparation\" class=\"anchor\" aria-label=\"Permalink: Dataset preparation\" href=\"#dataset-preparation\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDownload and generate mnist TFRecord\u003c/h2\u003e\u003ca id=\"user-content-download-and-generate-mnist-tfrecord\" class=\"anchor\" aria-label=\"Permalink: Download and generate mnist TFRecord\" href=\"#download-and-generate-mnist-tfrecord\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/slim\n# Generate train-mnist.tfrecord and test-mnist.tfrecord\nexport DATA_PATH=\u0026quot;${HOME}/dataset/mnist-data/\u0026quot;\npython download_and_convert_data.py --dataset_name mnist --dataset_dir ${DATA_PATH}\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/slim\n# Generate train-mnist.tfrecord and test-mnist.tfrecord\nexport DATA_PATH=\"${HOME}/dataset/mnist-data/\"\npython download_and_convert_data.py --dataset_name mnist --dataset_dir ${DATA_PATH}\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDownload and generate cifar-10 TFRecord\u003c/h2\u003e\u003ca id=\"user-content-download-and-generate-cifar-10-tfrecord\" class=\"anchor\" aria-label=\"Permalink: Download and generate cifar-10 TFRecord\" href=\"#download-and-generate-cifar-10-tfrecord\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/slim\n# Generate train-cifar10.tfrecord and test-cifar10.tfrecord\nexport DATA_PATH=\u0026quot;${HOME}/dataset/cifar10-data/\u0026quot; # the directory of database\npython download_and_convert_data.py --dataset_name cifar10 --dataset_dir ${DATA_PATH}\n\n# Instead of putting all training examples in one tfrecord file, we can split them by enabling --shard\n# This is useful for distributed training by date parallelsim, where we should split data across nodes\n# Generate train-xxxxx-of-xxxxx (1000 shards in default) and test-00000-of-00001 tfrecord shards\nexport DATA_PATH=\u0026quot;${HOME}/dataset/cifar10-shard-data/\u0026quot; # the directory of database\npython download_and_convert_data.py \\\n--dataset_name cifar10 \\\n--dataset_dir ${DATA_PATH} \\\n--shard True\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/slim\n# Generate train-cifar10.tfrecord and test-cifar10.tfrecord\nexport DATA_PATH=\"${HOME}/dataset/cifar10-data/\" # the directory of database\npython download_and_convert_data.py --dataset_name cifar10 --dataset_dir ${DATA_PATH}\n\n# Instead of putting all training examples in one tfrecord file, we can split them by enabling --shard\n# This is useful for distributed training by date parallelsim, where we should split data across nodes\n# Generate train-xxxxx-of-xxxxx (1000 shards in default) and test-00000-of-00001 tfrecord shards\nexport DATA_PATH=\"${HOME}/dataset/cifar10-shard-data/\" # the directory of database\npython download_and_convert_data.py \\\n--dataset_name cifar10 \\\n--dataset_dir ${DATA_PATH} \\\n--shard True\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDownload and generate ImageNet TFRecord\u003c/h2\u003e\u003ca id=\"user-content-download-and-generate-imagenet-tfrecord\" class=\"anchor\" aria-label=\"Permalink: Download and generate ImageNet TFRecord\" href=\"#download-and-generate-imagenet-tfrecord\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eBefore generating, \u003ccode\u003eRAW_PIXEL=True\u003c/code\u003e in \u003ccode\u003e${TERNGRAD_ROOT}/terngrad/inception/data/download_and_preprocess_imagenet.sh\u003c/code\u003e can enable storing raw RGB pixels of images into TFRecord.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eStoring raw pixels can save JPG decoding time but burden storage read bandwidth. Set \u003ccode\u003eRAW_PIXEL=True\u003c/code\u003e if high-speed external storage (like SSD) is used but decoder like in CPU cannot feed as fast as training (like in multi-GPUs).\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWhen \u003ccode\u003eRAW_PIXEL=True\u003c/code\u003e, setting \u003ccode\u003eRESIZE_DIMEN\u003c/code\u003e to a positive value enables image resizing before writing them to TFRecord files.\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"# location of where to place the ImageNet data\n# If ILSVRC2012_img_train.tar and ILSVRC2012_img_val.tar were downloaded before, \n# copy them in ${DATA_DIR} to save time\nDATA_DIR=/tmp/\n\n# build the preprocessing script.\nbazel build inception/download_and_preprocess_imagenet\n\n# run it\nbazel-bin/inception/download_and_preprocess_imagenet \u0026quot;${DATA_DIR}\u0026quot;\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e# location of where to place the ImageNet data\n# If ILSVRC2012_img_train.tar and ILSVRC2012_img_val.tar were downloaded before, \n# copy them in ${DATA_DIR} to save time\nDATA_DIR=/tmp/\n\n# build the preprocessing script.\nbazel build inception/download_and_preprocess_imagenet\n\n# run it\nbazel-bin/inception/download_and_preprocess_imagenet \"${DATA_DIR}\"\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eExamples on multi-gpu mode\u003c/h1\u003e\u003ca id=\"user-content-examples-on-multi-gpu-mode\" class=\"anchor\" aria-label=\"Permalink: Examples on multi-gpu mode\" href=\"#examples-on-multi-gpu-mode\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTraining CifarNet by TernGrad with Adam\u003c/h2\u003e\u003ca id=\"user-content-training-cifarnet-by-terngrad-with-adam\" class=\"anchor\" aria-label=\"Permalink: Training CifarNet by TernGrad with Adam\" href=\"#training-cifarnet-by-terngrad-with-adam\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/terngrad\n./run_multi_gpus_cifar10.sh\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/terngrad\n./run_multi_gpus_cifar10.sh\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_multi_gpus_cifar10.sh#L5-L33\"\u003erun_multi_gpus_cifar10.sh\u003c/a\u003e is a training script on cifar-10, which\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ecreates a subfolder under \u003ccode\u003e$ROOT_WORKSPACE/${DATASET_NAME}_xxx/\u003c/code\u003e to store the training data (\u003ccode\u003e${ROOT_WORKSPACE}/${DATASET_NAME}_training_data/\u003c/code\u003e), evaluating data (\u003ccode\u003e${ROOT_WORKSPACE}/${DATASET_NAME}_eval_data/\u003c/code\u003e) and logs (\u003ccode\u003e${ROOT_WORKSPACE}/${DATASET_NAME}_info/\u003c/code\u003e). The subfolder name or log filename is similar to \u003ccode\u003ecifar10_cifar10_alexnet_24_adam_1_0.0002_2.5_0_0.004_0.9_1_128_2_Tue_Sep_19_15-27-51_EDT_2017\u003c/code\u003e.\u003c/li\u003e\n\u003cli\u003estarts training and\u003c/li\u003e\n\u003cli\u003estarts evaluating.\u003cbr\u003e\nYou can change those environments to play.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eUse \u003ccode\u003e--help\u003c/code\u003e to check descriptions for usage of python executables. For example,\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"bazel-bin/inception/cifar10_train --help\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ebazel-bin/inception/cifar10_train --help\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eSome important configurations related to TernGrad:\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"--size_to_binarize SIZE_TO_BINARIZE\n The min number of parameters in a Variable (weights/biases) to enable ternarizing this Variable. \n 1 means ternarizing all. You can use this to exclude some small Variables\n--num_gpus NUM_GPUS \n How many GPUs to use.\n--num_nodes NUM_NODES\n How many virtual nodes to use. One GPU can have multiple nodes \n This enables emulating N workers in M GPUs, where N \u0026gt; M\n This is a good feature to verify TernGrad algorithm when multiple GPUs are unavailable\n--grad_bits [32/1]\n The number of gradient bits. Either 32 or 1. 32 for floating, and 1 for terngrad \n (I know ternary is not 1 bit. This is just an argument to use either floating or terngrad. \n We may consider to extend it to more options ranging from terngrad to floating, \n like 2 bits, 4 bits, 8 bits, etc)\n--clip_factor CLIP_FACTOR\n The factor of stddev to clip gradients (0.0 means no clipping). \n This is the value of c in gradient clipping technique.\n 2.5 works well in general\n--quantize_logits [True/False]\n--noquantize_logits \n If quantize the gradients in the last logits layer. \n (sometimes, a skew distribution of gradients in last layer may affect the effectiveness of terngrad.\n asymmetric ternary levels may be more effective)\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e--size_to_binarize SIZE_TO_BINARIZE\n The min number of parameters in a Variable (weights/biases) to enable ternarizing this Variable. \n 1 means ternarizing all. You can use this to exclude some small Variables\n--num_gpus NUM_GPUS \n How many GPUs to use.\n--num_nodes NUM_NODES\n How many virtual nodes to use. One GPU can have multiple nodes \n This enables emulating N workers in M GPUs, where N \u0026gt; M\n This is a good feature to verify TernGrad algorithm when multiple GPUs are unavailable\n--grad_bits [32/1]\n The number of gradient bits. Either 32 or 1. 32 for floating, and 1 for terngrad \n (I know ternary is not 1 bit. This is just an argument to use either floating or terngrad. \n We may consider to extend it to more options ranging from terngrad to floating, \n like 2 bits, 4 bits, 8 bits, etc)\n--clip_factor CLIP_FACTOR\n The factor of stddev to clip gradients (0.0 means no clipping). \n This is the value of c in gradient clipping technique.\n 2.5 works well in general\n--quantize_logits [True/False]\n--noquantize_logits \n If quantize the gradients in the last logits layer. \n (sometimes, a skew distribution of gradients in last layer may affect the effectiveness of terngrad.\n asymmetric ternary levels may be more effective)\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eMore explanations are also covered in \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_multi_gpus_cifar10.sh#L5-L33\"\u003erun_multi_gpus_cifar10.sh\u003c/a\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eMore training bash scripts are in \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad\"\u003eterngrad\u003c/a\u003e, which have similar arguments.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eExamples on distributed-node mode\u003c/h1\u003e\u003ca id=\"user-content-examples-on-distributed-node-mode\" class=\"anchor\" aria-label=\"Permalink: Examples on distributed-node mode\" href=\"#examples-on-distributed-node-mode\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003essh setup\u003c/h2\u003e\u003ca id=\"user-content-ssh-setup\" class=\"anchor\" aria-label=\"Permalink: ssh setup\" href=\"#ssh-setup\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe provide a single script to remotely launch all workers and parameter servers.\nTo authorize access to each other, all machines must share the same ssh key. Please follow this \u003ca href=\"https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/\"\u003etutorial\u003c/a\u003e to setup. Moreover, make sure you have added the content in the public key (\u003ccode\u003e~/.ssh/id_rsa.pub\u003c/code\u003e) to \u003ccode\u003e~/.ssh/authorized_keys\u003c/code\u003e. In some systems, you also need to add the public key to \u003ccode\u003e~/.ssh/authorized_keys2\u003c/code\u003e. After this, you can simply copy keys (\u003ccode\u003e~/.ssh/id_rsa\u003c/code\u003e and \u003ccode\u003e~/.ssh/id_rsa.pub\u003c/code\u003e) and authorizing files (\u003ccode\u003e~/.ssh/authorized_keys\u003c/code\u003e and \u003ccode\u003e~/.ssh/authorized_keys2\u003c/code\u003e) to all machines.\nAt the first run, you may need to answer \u003ccode\u003eyes\u003c/code\u003e to\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"The authenticity of host '10.236.176.29 (10.236.176.29)' can't be established.\nECDSA key fingerprint is SHA256:jkfjkdslajfklsjaflkjs/jowufuf98e8e8eu9.\nAre you sure you want to continue connecting (yes/no)?\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003eThe authenticity of host '10.236.176.29 (10.236.176.29)' can't be established.\nECDSA key fingerprint is SHA256:jkfjkdslajfklsjaflkjs/jowufuf98e8e8eu9.\nAre you sure you want to continue connecting (yes/no)?\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eOur bash script uses \u003ccode\u003essh\u003c/code\u003e to login and start worker/ps. If some variables are configured in \u003ccode\u003e.bashrc\u003c/code\u003e and are necessary for training (e.g., the \u003ccode\u003ePATH\u003c/code\u003e of anaconda), you may need to source \u003ccode\u003e~/.bashrc\u003c/code\u003e in \u003ccode\u003e~/.bash_profile\u003c/code\u003e or \u003ccode\u003e~/.profile\u003c/code\u003e by adding\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"if [ -f ~/.bashrc ]; then\n . ~/.bashrc\nfi\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003eif [ -f ~/.bashrc ]; then\n . ~/.bashrc\nfi\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIn some linux distributions (e.g. ubuntu) you may need to comment\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"# If not running interactively, don't do anything\ncase $- in\n *i*) ;;\n *) return;;\nesac\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e# If not running interactively, don't do anything\ncase $- in\n *i*) ;;\n *) return;;\nesac\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eA toy example\u003c/h2\u003e\u003ca id=\"user-content-a-toy-example\" class=\"anchor\" aria-label=\"Permalink: A toy example\" href=\"#a-toy-example\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_dist_cifar10.sh\"\u003erun_dist_cifar10.sh\u003c/a\u003e is a toy example by launching one parameter server and two workers in \u003ccode\u003elocalhost\u003c/code\u003e.\nBefore start, we must split cifar-10 dataset to two parts:\n\u003ccode\u003e$HOME/dataset/cifar10-data-shard-500-999\u003c/code\u003e and \u003ccode\u003e$HOME/dataset/cifar10-data-shard-0-499\u003c/code\u003e, which each worker paralell fetches and trains its model replica.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe python executable is \u003ccode\u003ebazel-bin/inception/cifar10_distributed_train\u003c/code\u003e, of which most arguments are similar to \u003ccode\u003ebazel-bin/inception/cifar10_train\u003c/code\u003e for multi-gpu mode but with\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"--job_name JOB_NAME One of \u0026quot;ps\u0026quot;, \u0026quot;worker\u0026quot;\n--task_id TASK_ID Task ID of the worker/replica running the training.\n--ps_hosts PS_HOSTS Comma-separated list of hostname:port for the\n parameter server jobs. e.g.\n 'machine1:2222,machine2:1111,machine2:2222'\n--worker_hosts WORKER_HOSTS\n Comma-separated list of hostname:port for the worker\n jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222'\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e--job_name JOB_NAME One of \"ps\", \"worker\"\n--task_id TASK_ID Task ID of the worker/replica running the training.\n--ps_hosts PS_HOSTS Comma-separated list of hostname:port for the\n parameter server jobs. e.g.\n 'machine1:2222,machine2:1111,machine2:2222'\n--worker_hosts WORKER_HOSTS\n Comma-separated list of hostname:port for the worker\n jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222'\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eFor more details, type \u003ccode\u003ebazel-bin/inception/cifar10_distributed_train --help\u003c/code\u003e or go \u003ca href=\"#backup-inception-in-tensorflow\"\u003ehere\u003c/a\u003e.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eA single script to launch all\u003c/h2\u003e\u003ca id=\"user-content-a-single-script-to-launch-all\" class=\"anchor\" aria-label=\"Permalink: A single script to launch all\" href=\"#a-single-script-to-launch-all\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eConfig \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/config_dist.sh\"\u003econfig_dist.sh\u003c/a\u003e and run \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_dist.sh\"\u003erun_dist.sh\u003c/a\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eYou only need to configure \u003ccode\u003econfig_dist.sh\u003c/code\u003e (including workers, ps, gpu devices and dataset paths), and write a \u003ccode\u003eWORKER_SCRIPT\u003c/code\u003e to specify how to start a worker. \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_single_worker_cifarnet.sh\"\u003erun_single_worker_cifarnet.sh\u003c/a\u003e and \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/run_single_worker_alexnet.sh\"\u003erun_single_worker_alexnet.sh\u003c/a\u003e are two \u003ccode\u003eWORKER_SCRIPT\u003c/code\u003e examples, which basically set hyperparameters and start training.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eUsage is explained within \u003ccode\u003econfig_dist.sh\u003c/code\u003e script.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBy default, results are saved in \u003ccode\u003e${HOME}/tmp/\u003c/code\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWe also provide \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/split_dataset.sh\"\u003esplit_dataset.sh\u003c/a\u003e to \u003cem\u003elocally\u003c/em\u003e split shards of training dataset. Usage: \u003ccode\u003e./split_dataset.sh \u0026lt;path-of-dataset-to-be-split\u0026gt; \u0026lt;total_workers\u0026gt; \u0026lt;worker_index\u0026gt;\u003c/code\u003e.\nIt will create a subfolder under \u003ccode\u003e\u0026lt;path-of-dataset-to-be-split\u0026gt;\u003c/code\u003e named as \u003ccode\u003eworker_\u0026lt;worker_index\u0026gt;_of_\u0026lt;total_workers\u0026gt;\u003c/code\u003e, and create links to shard files belonging to this worker.\nFor example,\u003c/p\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"$ cd ~/dataset/imagenet-data/\n$ ls -l\n -rw-rw-r-- 1 wew57 wew57 144582762 Mar 24 2017 train-00000-of-01024\n -rw-rw-r-- 1 wew57 wew57 148475588 Mar 24 2017 train-00001-of-01024\n -rw-rw-r-- 1 wew57 wew57 150196808 Mar 24 2017 train-00002-of-01024\n ...\n -rw-rw-r-- 1 wew57 wew57 144180160 Mar 24 2017 train-01021-of-01024\n -rw-rw-r-- 1 wew57 wew57 140903282 Mar 24 2017 train-01022-of-01024\n -rw-rw-r-- 1 wew57 wew57 138485470 Mar 24 2017 train-01023-of-01024\n$ cd /home/wew57/github/users/wenwei202/terngrad/terngrad\n$ ./split_dataset.sh ~/dataset/imagenet-data/ 16 1\n Splitting to /home/wew57/dataset/imagenet-data//worker_1_of_16 ...\n$ ls -l /home/wew57/dataset/imagenet-data//worker_1_of_16\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00064-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00064-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00065-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00065-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00066-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00066-of-01024\n ...\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00125-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00125-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00126-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00126-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00127-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00127-of-01024\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003e$ cd ~/dataset/imagenet-data/\n$ ls -l\n -rw-rw-r-- 1 wew57 wew57 144582762 Mar 24 2017 train-00000-of-01024\n -rw-rw-r-- 1 wew57 wew57 148475588 Mar 24 2017 train-00001-of-01024\n -rw-rw-r-- 1 wew57 wew57 150196808 Mar 24 2017 train-00002-of-01024\n ...\n -rw-rw-r-- 1 wew57 wew57 144180160 Mar 24 2017 train-01021-of-01024\n -rw-rw-r-- 1 wew57 wew57 140903282 Mar 24 2017 train-01022-of-01024\n -rw-rw-r-- 1 wew57 wew57 138485470 Mar 24 2017 train-01023-of-01024\n$ cd /home/wew57/github/users/wenwei202/terngrad/terngrad\n$ ./split_dataset.sh ~/dataset/imagenet-data/ 16 1\n Splitting to /home/wew57/dataset/imagenet-data//worker_1_of_16 ...\n$ ls -l /home/wew57/dataset/imagenet-data//worker_1_of_16\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00064-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00064-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00065-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00065-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00066-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00066-of-01024\n ...\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00125-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00125-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00126-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00126-of-01024\n lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00127-of-01024 -\u0026gt; /home/wew57/dataset/imagenet-data//train-00127-of-01024\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eYou can stop all tasks by \u003ca href=\"/wenwei202/terngrad/blob/master/terngrad/stop_dist.sh\"\u003estop_dist.sh\u003c/a\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eCurrently, distributed-node mode only supports 32bit gradients. It will take a while to hack the highly-encapsulated \u003ccode\u003eSyncReplicasOptimizer\u003c/code\u003e to integrate TernGrad. Keep updating.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePython executables\u003c/h1\u003e\u003ca id=\"user-content-python-executables\" class=\"anchor\" aria-label=\"Permalink: Python executables\" href=\"#python-executables\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eBash scripts essentially call python executables. We list python commands here for agile development.\nTaking 32bit gradients as examples.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNode that, in TernGrad, parameters are allocated in each GPU to reduce communication because we can communicate quantized gradients instead of floating parameters.\nBy default, the program saves parameters in all GPUs. To evaluate/test, use \u003ccode\u003e--tower \u0026lt;gpu_id\u0026gt;\u003c/code\u003e to specify which GPU's parameters you want to test on. (We will try to remove this feature to save storage, because the parameter sets are identical across all GPUs). Alteratively, you can use \u003ccode\u003e--save_tower 0\u003c/code\u003e in training executables to avoid saving duplicated parameters, in which case, \u003ccode\u003e--tower \u0026lt;gpu_id\u0026gt;\u003c/code\u003e is unnecessary during evaluation/testing.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBuild and run evaluating/training LeNet on mnist\u003c/h2\u003e\u003ca id=\"user-content-build-and-run-evaluatingtraining-lenet-on-mnist\" class=\"anchor\" aria-label=\"Permalink: Build and run evaluating/training LeNet on mnist\" href=\"#build-and-run-evaluatingtraining-lenet-on-mnist\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/mnist_train\nbazel build inception/mnist_eval\n\nbazel-bin/inception/mnist_train \\\n--optimizer momentum \\\n--initial_learning_rate 0.01 \\\n--learning_rate_decay_type polynomial \\\n--max_steps 10000 \\\n--net lenet \\\n--image_size 28 \\\n--num_gpus 2 \\\n--batch_size 64 \\\n--train_dir /tmp/mnist_train \\\n--data_dir ~/dataset/mnist-data/\n\nbazel-bin/inception/mnist_eval \\\n--data_dir ~/dataset/mnist-data/ \\\n--net lenet \\\n--image_size 28 \\\n--batch_size 100 \\\n--checkpoint_dir /tmp/mnist_train \\\n--restore_avg_var True \\\n--eval_interval_secs 300 \\\n--eval_dir /tmp/mnist_eval \\\n--subset test\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/mnist_train\nbazel build inception/mnist_eval\n\nbazel-bin/inception/mnist_train \\\n--optimizer momentum \\\n--initial_learning_rate 0.01 \\\n--learning_rate_decay_type polynomial \\\n--max_steps 10000 \\\n--net lenet \\\n--image_size 28 \\\n--num_gpus 2 \\\n--batch_size 64 \\\n--train_dir /tmp/mnist_train \\\n--data_dir ~/dataset/mnist-data/\n\nbazel-bin/inception/mnist_eval \\\n--data_dir ~/dataset/mnist-data/ \\\n--net lenet \\\n--image_size 28 \\\n--batch_size 100 \\\n--checkpoint_dir /tmp/mnist_train \\\n--restore_avg_var True \\\n--eval_interval_secs 300 \\\n--eval_dir /tmp/mnist_eval \\\n--subset test\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBuild and run evaluating/training on cifar-10\u003c/h2\u003e\u003ca id=\"user-content-build-and-run-evaluatingtraining-on-cifar-10\" class=\"anchor\" aria-label=\"Permalink: Build and run evaluating/training on cifar-10\" href=\"#build-and-run-evaluatingtraining-on-cifar-10\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/cifar10_train\nbazel build inception/cifar10_eval\n\nbazel-bin/inception/cifar10_train \\\n--optimizer adam \\\n--initial_learning_rate 0.0002 \\\n--num_epochs_per_decay 256 \\\n--max_steps 200000 \\\n--net cifar10_alexnet \\\n--image_size 24 \\\n--num_gpus 2 \\\n--batch_size 128 \\\n--train_dir /tmp/cifar10_train \\\n--data_dir ~/dataset/cifar10-data/ \n\nbazel-bin/inception/cifar10_eval \\\n--data_dir ~/dataset/cifar10-data/ \\\n--net cifar10_alexnet \\\n--image_size 24 \\\n--batch_size 50 \\\n--checkpoint_dir /tmp/cifar10_train \\\n--restore_avg_var True \\\n--eval_interval_secs 300 \\\n--eval_dir /tmp/cifar10_eval \\\n--subset test\n\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/cifar10_train\nbazel build inception/cifar10_eval\n\nbazel-bin/inception/cifar10_train \\\n--optimizer adam \\\n--initial_learning_rate 0.0002 \\\n--num_epochs_per_decay 256 \\\n--max_steps 200000 \\\n--net cifar10_alexnet \\\n--image_size 24 \\\n--num_gpus 2 \\\n--batch_size 128 \\\n--train_dir /tmp/cifar10_train \\\n--data_dir ~/dataset/cifar10-data/ \n\nbazel-bin/inception/cifar10_eval \\\n--data_dir ~/dataset/cifar10-data/ \\\n--net cifar10_alexnet \\\n--image_size 24 \\\n--batch_size 50 \\\n--checkpoint_dir /tmp/cifar10_train \\\n--restore_avg_var True \\\n--eval_interval_secs 300 \\\n--eval_dir /tmp/cifar10_eval \\\n--subset test\n\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBuild and run ImageNet\u003c/h2\u003e\u003ca id=\"user-content-build-and-run-imagenet\" class=\"anchor\" aria-label=\"Permalink: Build and run ImageNet\" href=\"#build-and-run-imagenet\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"snippet-clipboard-content notranslate position-relative overflow-auto\" data-snippet-clipboard-copy-content=\"cd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/imagenet_train\nbazel build inception/imagenet_eval\n\nbazel-bin/inception/imagenet_train \\\n--optimizer momentum \\\n--net alexnet \\\n--image_size 224 \\\n--num_gpus 2 \\\n--batch_size 256 \\\n--train_dir /tmp/imagenet_train \\\n--data_dir ~/dataset/imagenet-data/\n\n\nbazel-bin/inception/imagenet_eval \\\n--data_dir ~/dataset/imagenet-data/ \\\n--net alexnet \\\n--image_size 224 \\\n--batch_size 50 \\\n--checkpoint_dir /tmp/imagenet_train \\\n--restore_avg_var True \\\n--eval_dir /tmp/imagenet_eval\n\"\u003e\u003cpre class=\"notranslate\"\u003e\u003ccode\u003ecd ${TERNGRAD_ROOT}/terngrad\nbazel build inception/imagenet_train\nbazel build inception/imagenet_eval\n\nbazel-bin/inception/imagenet_train \\\n--optimizer momentum \\\n--net alexnet \\\n--image_size 224 \\\n--num_gpus 2 \\\n--batch_size 256 \\\n--train_dir /tmp/imagenet_train \\\n--data_dir ~/dataset/imagenet-data/\n\n\nbazel-bin/inception/imagenet_eval \\\n--data_dir ~/dataset/imagenet-data/ \\\n--net alexnet \\\n--image_size 224 \\\n--batch_size 50 \\\n--checkpoint_dir /tmp/imagenet_train \\\n--restore_avg_var True \\\n--eval_dir /tmp/imagenet_eval\n\n\u003c/code\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eOpen questions\u003c/h1\u003e\u003ca id=\"user-content-open-questions\" class=\"anchor\" aria-label=\"Permalink: Open questions\" href=\"#open-questions\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003col dir=\"auto\"\u003e\n\u003cli\u003eHow will TernGrad work with asynchronous SGD\u003c/li\u003e\n\u003cli\u003eHow to reduce variance of TernGrad when the larger variance introduces some accuracy loss\u003c/li\u003e\n\u003cli\u003eHow will TernGrad work when server-to-worker gradients are ternarized in the same way\u003c/li\u003e\n\u003c/ol\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch1 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eBackup (Inception in TensorFlow)\u003c/h1\u003e\u003ca id=\"user-content-backup-inception-in-tensorflow\" class=\"anchor\" aria-label=\"Permalink: Backup (Inception in TensorFlow)\" href=\"#backup-inception-in-tensorflow\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"http://www.image-net.org/\" rel=\"nofollow\"\u003eImageNet\u003c/a\u003e is a common academic data set in machine\nlearning for training an image recognition system. Code in this directory\ndemonstrates how to use TensorFlow to train and evaluate a type of convolutional\nneural network (CNN) on this academic data set. In particular, we demonstrate\nhow to train the Inception v3 architecture as specified in:\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cem\u003eRethinking the Inception Architecture for Computer Vision\u003c/em\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eChristian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew\nWojna\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"http://arxiv.org/abs/1512.00567\" rel=\"nofollow\"\u003ehttp://arxiv.org/abs/1512.00567\u003c/a\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThis network achieves 21.2% top-1 and 5.6% top-5 error for single frame\nevaluation with a computational cost of 5 billion multiply-adds per inference\nand with using less than 25 million parameters. Below is a visualization of the\nmodel architecture.\u003c/p\u003e\n\n\n\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eDescription of Code\u003c/h2\u003e\u003ca id=\"user-content-description-of-code\" class=\"anchor\" aria-label=\"Permalink: Description of Code\" href=\"#description-of-code\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe code base provides three core binaries for:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eTraining an Inception v3 network from scratch across multiple GPUs and/or\nmultiple machines using the ImageNet 2012 Challenge training data set.\u003c/li\u003e\n\u003cli\u003eEvaluating an Inception v3 network using the ImageNet 2012 Challenge\nvalidation data set.\u003c/li\u003e\n\u003cli\u003eRetraining an Inception v3 network on a novel task and back-propagating the\nerrors to fine tune the network weights.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThe training procedure employs synchronous stochastic gradient descent across\nmultiple GPUs. The user may specify the number of GPUs they wish harness. The\nsynchronous training performs \u003cem\u003ebatch-splitting\u003c/em\u003e by dividing a given batch across\nmultiple GPUs.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe training set up is nearly identical to the section [Training a Model Using\nMultiple GPU Cards]\n(\u003ca href=\"https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards\u003c/a\u003e)\nwhere we have substituted the CIFAR-10 model architecture with Inception v3. The\nprimary differences with that setup are:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eCalculate and update the batch-norm statistics during training so that they\nmay be substituted in during evaluation.\u003c/li\u003e\n\u003cli\u003eSpecify the model architecture using a (still experimental) higher level\nlanguage called TensorFlow-Slim.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eFor more details about TensorFlow-Slim, please see the [Slim README]\n(inception/slim/README.md). Please note that this higher-level language is still\n\u003cem\u003eexperimental\u003c/em\u003e and the API may change over time depending on usage and\nsubsequent research.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eGetting Started\u003c/h2\u003e\u003ca id=\"user-content-getting-started\" class=\"anchor\" aria-label=\"Permalink: Getting Started\" href=\"#getting-started\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eNOTE\u003c/strong\u003e Before doing anything, we first need to build TensorFlow from source,\nand installed as a PIP package. Please follow the instructions at [Installing\nFrom Source]\n(\u003ca href=\"https://www.tensorflow.org/get_started/os_setup.html#create-the-pip-package-and-install\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/get_started/os_setup.html#create-the-pip-package-and-install\u003c/a\u003e).\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBefore you run the training script for the first time, you will need to download\nand convert the ImageNet data to native TFRecord format. The TFRecord format\nconsists of a set of sharded files where each entry is a serialized \u003ccode\u003etf.Example\u003c/code\u003e\nproto. Each \u003ccode\u003etf.Example\u003c/code\u003e proto contains the ImageNet image (JPEG encoded) as\nwell as metadata such as label and bounding box information. See\n\u003ca href=\"/wenwei202/terngrad/blob/master/inception/image_processing.py\"\u003e\u003ccode\u003eparse_example_proto\u003c/code\u003e\u003c/a\u003e for details.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWe provide a single \u003ca href=\"/wenwei202/terngrad/blob/master/inception/data/download_and_preprocess_imagenet.sh\"\u003escript\u003c/a\u003e for\ndownloading and converting ImageNet data to TFRecord format. Downloading and\npreprocessing the data may take several hours (up to half a day) depending on\nyour network and computer speed. Please be patient.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eTo begin, you will need to sign up for an account with [ImageNet]\n(\u003ca href=\"http://image-net.org\" rel=\"nofollow\"\u003ehttp://image-net.org\u003c/a\u003e) to gain access to the data. Look for the sign up page,\ncreate an account and request an access key to download the data.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAfter you have \u003ccode\u003eUSERNAME\u003c/code\u003e and \u003ccode\u003ePASSWORD\u003c/code\u003e, you are ready to run our script. Make\nsure that your hard disk has at least 500 GB of free space for downloading and\nstoring the data. Here we select \u003ccode\u003eDATA_DIR=$HOME/dataset/imagenet-data\u003c/code\u003e as such a\nlocation but feel free to edit accordingly.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWhen you run the below script, please enter \u003cem\u003eUSERNAME\u003c/em\u003e and \u003cem\u003ePASSWORD\u003c/em\u003e when\nprompted. This will occur at the very beginning. Once these values are entered,\nyou will not need to interact with the script again.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# location of where to place the ImageNet data\nDATA_DIR=$HOME/dataset/imagenet-data\n\n# build the preprocessing script.\nbazel build inception/download_and_preprocess_imagenet\n\n# run it\nbazel-bin/inception/download_and_preprocess_imagenet \u0026quot;${DATA_DIR}\u0026quot;\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e location of where to place the ImageNet data\u003c/span\u003e\nDATA_DIR=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/dataset/imagenet-data\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e build the preprocessing script.\u003c/span\u003e\nbazel build inception/download_and_preprocess_imagenet\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e run it\u003c/span\u003e\nbazel-bin/inception/download_and_preprocess_imagenet \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${DATA_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe final line of the output script should read:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"2016-02-17 14:30:17.287989: Finished writing all 1281167 images in data set.\"\u003e\u003cpre\u003e2016-02-17 14:30:17.287989: Finished writing all 1281167 images \u003cspan class=\"pl-k\"\u003ein\u003c/span\u003e data set.\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWhen the script finishes you will find 1024 and 128 training and validation\nfiles in the \u003ccode\u003eDATA_DIR\u003c/code\u003e. The files will match the patterns \u003ccode\u003etrain-????-of-1024\u003c/code\u003e\nand \u003ccode\u003evalidation-?????-of-00128\u003c/code\u003e, respectively.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.youtube.com/watch?v=9bZkp7q19f0\" rel=\"nofollow\"\u003eCongratulations!\u003c/a\u003e You are now\nready to train or evaluate with the ImageNet data set.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Train from Scratch\u003c/h2\u003e\u003ca id=\"user-content-how-to-train-from-scratch\" class=\"anchor\" aria-label=\"Permalink: How to Train from Scratch\" href=\"#how-to-train-from-scratch\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eWARNING\u003c/strong\u003e Training an Inception v3 network from scratch is a computationally\nintensive task and depending on your compute setup may take several days or even\nweeks.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cem\u003eBefore proceeding\u003c/em\u003e please read the [Convolutional Neural Networks]\n(\u003ca href=\"https://www.tensorflow.org/tutorials/deep_cnn/index.html\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/tutorials/deep_cnn/index.html\u003c/a\u003e) tutorial in\nparticular focus on [Training a Model Using Multiple GPU Cards]\n(\u003ca href=\"https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards\u003c/a\u003e)\n. The model training method is nearly identical to that described in the\nCIFAR-10 multi-GPU model training. Briefly, the model training\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003ePlaces an individual model replica on each GPU. Split the batch across the\nGPUs.\u003c/li\u003e\n\u003cli\u003eUpdates model parameters synchronously by waiting for all GPUs to finish\nprocessing a batch of data.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThe training procedure is encapsulated by this diagram of how operations and\nvariables are placed on CPU and GPUs respectively.\u003c/p\u003e\n\u003cdiv dir=\"auto\"\u003e\n \u003ca target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://camo.githubusercontent.com/bc2f3dcd0ff224146babb58aff157064dccf7eb320a72533a52f0c7d2981b3fb/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f506172616c6c656c69736d2e706e67\"\u003e\u003cimg style=\"width: 100%; max-width: 100%;\" src=\"https://camo.githubusercontent.com/bc2f3dcd0ff224146babb58aff157064dccf7eb320a72533a52f0c7d2981b3fb/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f506172616c6c656c69736d2e706e67\" data-canonical-src=\"https://www.tensorflow.org/images/Parallelism.png\"\u003e\u003c/a\u003e\n\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eEach tower computes the gradients for a portion of the batch and the gradients\nare combined and averaged across the multiple towers in order to provide a\nsingle update of the Variables stored on the CPU.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eA crucial aspect of training a network of this size is \u003cem\u003etraining speed\u003c/em\u003e in terms\nof wall-clock time. The training speed is dictated by many factors -- most\nimportantly the batch size and the learning rate schedule. Both of these\nparameters are heavily coupled to the hardware set up.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eGenerally speaking, a batch size is a difficult parameter to tune as it requires\nbalancing memory demands of the model, memory available on the GPU and speed of\ncomputation. Generally speaking, employing larger batch sizes leads to more\nefficient computation and potentially more efficient training steps.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWe have tested several hardware setups for training this model from scratch but\nwe emphasize that depending your hardware set up, you may need to adapt the\nbatch size and learning rate schedule.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003ePlease see the comments in \u003ccode\u003einception_train.py\u003c/code\u003e for a few selected learning rate\nplans based on some selected hardware setups.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eTo train this model, you simply need to specify the following:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/imagenet_train\n\n# run it\nbazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/tmp/imagenet_train --data_dir=/tmp/imagenet_data\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/imagenet_train\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e run it\u003c/span\u003e\nbazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/tmp/imagenet_train --data_dir=/tmp/imagenet_data\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe model reads in the ImageNet training data from \u003ccode\u003e--data_dir\u003c/code\u003e. If you followed\nthe instructions in \u003ca href=\"#getting-started\"\u003eGetting Started\u003c/a\u003e, then set\n\u003ccode\u003e--data_dir=\"${DATA_DIR}\"\u003c/code\u003e. The script assumes that there exists a set of\nsharded TFRecord files containing the ImageNet data. If you have not created\nTFRecord files, please refer to \u003ca href=\"#getting-started\"\u003eGetting Started\u003c/a\u003e\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eHere is the output of the above command line when running on a Tesla K40c:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"2016-03-07 12:24:59.922898: step 0, loss = 13.11 (5.3 examples/sec; 6.064 sec/batch)\n2016-03-07 12:25:55.206783: step 10, loss = 13.71 (9.4 examples/sec; 3.394 sec/batch)\n2016-03-07 12:26:28.905231: step 20, loss = 14.81 (9.5 examples/sec; 3.380 sec/batch)\n2016-03-07 12:27:02.699719: step 30, loss = 14.45 (9.5 examples/sec; 3.378 sec/batch)\n2016-03-07 12:27:36.515699: step 40, loss = 13.98 (9.5 examples/sec; 3.376 sec/batch)\n2016-03-07 12:28:10.220956: step 50, loss = 13.92 (9.6 examples/sec; 3.327 sec/batch)\n2016-03-07 12:28:43.658223: step 60, loss = 13.28 (9.6 examples/sec; 3.350 sec/batch)\n...\"\u003e\u003cpre\u003e2016-03-07 12:24:59.922898: step 0, loss = 13.11 (5.3 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 6.064 sec/batch)\n2016-03-07 12:25:55.206783: step 10, loss = 13.71 (9.4 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.394 sec/batch)\n2016-03-07 12:26:28.905231: step 20, loss = 14.81 (9.5 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.380 sec/batch)\n2016-03-07 12:27:02.699719: step 30, loss = 14.45 (9.5 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.378 sec/batch)\n2016-03-07 12:27:36.515699: step 40, loss = 13.98 (9.5 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.376 sec/batch)\n2016-03-07 12:28:10.220956: step 50, loss = 13.92 (9.6 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.327 sec/batch)\n2016-03-07 12:28:43.658223: step 60, loss = 13.28 (9.6 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 3.350 sec/batch)\n...\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIn this example, a log entry is printed every 10 step and the line includes the\ntotal loss (starts around 13.0-14.0) and the speed of processing in terms of\nthroughput (examples / sec) and batch speed (sec/batch).\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe number of GPU devices is specified by \u003ccode\u003e--num_gpus\u003c/code\u003e (which defaults to 1).\nSpecifying \u003ccode\u003e--num_gpus\u003c/code\u003e greater then 1 splits the batch evenly split across the\nGPU cards.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/imagenet_train\n\n# run it\nbazel-bin/inception/imagenet_train --num_gpus=2 --batch_size=64 --train_dir=/tmp/imagenet_train\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/imagenet_train\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e run it\u003c/span\u003e\nbazel-bin/inception/imagenet_train --num_gpus=2 --batch_size=64 --train_dir=/tmp/imagenet_train\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThis model splits the batch of 64 images across 2 GPUs and calculates the\naverage gradient by waiting for both GPUs to finish calculating the gradients\nfrom their respective data (See diagram above). Generally speaking, using larger\nnumbers of GPUs leads to higher throughput as well as the opportunity to use\nlarger batch sizes. In turn, larger batch sizes imply better estimates of the\ngradient enabling the usage of higher learning rates. In summary, using more\nGPUs results in simply faster training speed.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNote that selecting a batch size is a difficult parameter to tune as it requires\nbalancing memory demands of the model, memory available on the GPU and speed of\ncomputation. Generally speaking, employing larger batch sizes leads to more\nefficient computation and potentially more efficient training steps.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNote that there is considerable noise in the loss function on individual steps\nin the previous log. Because of this noise, it is difficult to discern how well\na model is learning. The solution to the last problem is to launch TensorBoard\npointing to the directory containing the events log.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"tensorboard --logdir=/tmp/imagenet_train\"\u003e\u003cpre\u003etensorboard --logdir=/tmp/imagenet_train\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eTensorBoard has access to the many Summaries produced by the model that describe\nmultitudes of statistics tracking the model behavior and the quality of the\nlearned model. In particular, TensorBoard tracks a exponentially smoothed\nversion of the loss. In practice, it is far easier to judge how well a model\nlearns by monitoring the smoothed version of the loss.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Train from Scratch in a Distributed Setting\u003c/h2\u003e\u003ca id=\"user-content-how-to-train-from-scratch-in-a-distributed-setting\" class=\"anchor\" aria-label=\"Permalink: How to Train from Scratch in a Distributed Setting\" href=\"#how-to-train-from-scratch-in-a-distributed-setting\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eNOTE\u003c/strong\u003e Distributed TensorFlow requires version 0.8 or later.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eDistributed TensorFlow lets us use multiple machines to train a model faster.\nThis is quite different from the training with multiple GPU towers on a single\nmachine where all parameters and gradients computation are in the same place. We\ncoordinate the computation across multiple machines by employing a centralized\nrepository for parameters that maintains a unified, single copy of model\nparameters. Each individual machine sends gradient updates to the centralized\nparameter repository which coordinates these updates and sends back updated\nparameters to the individual machines running the model training.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWe term each machine that runs a copy of the training a \u003ccode\u003eworker\u003c/code\u003e or \u003ccode\u003ereplica\u003c/code\u003e.\nWe term each machine that maintains model parameters a \u003ccode\u003eps\u003c/code\u003e, short for\n\u003ccode\u003eparameter server\u003c/code\u003e. Note that we might have more than one machine acting as a\n\u003ccode\u003eps\u003c/code\u003e as the model parameters may be sharded across multiple machines.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eVariables may be updated with synchronous or asynchronous gradient updates. One\nmay construct a an [\u003ccode\u003eOptimizer\u003c/code\u003e]\n(\u003ca href=\"https://www.tensorflow.org/api_docs/python/train.html#optimizers\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/api_docs/python/train.html#optimizers\u003c/a\u003e) in TensorFlow\nthat constructs the necessary graph for either case diagrammed below from\nTensorFlow [Whitepaper]\n(\u003ca href=\"http://download.tensorflow.org/paper/whitepaper2015.pdf\" rel=\"nofollow\"\u003ehttp://download.tensorflow.org/paper/whitepaper2015.pdf\u003c/a\u003e):\u003c/p\u003e\n\u003cdiv dir=\"auto\"\u003e\n \u003ca target=\"_blank\" rel=\"noopener noreferrer nofollow\" href=\"https://camo.githubusercontent.com/44e2ae57208da9a61ccb7d87bf0491a5ba6806f16e97e2511e57abe6e7e06e27/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f74656e736f72666c6f775f666967757265372e706e67\"\u003e\u003cimg style=\"width: 100%; max-width: 100%;\" src=\"https://camo.githubusercontent.com/44e2ae57208da9a61ccb7d87bf0491a5ba6806f16e97e2511e57abe6e7e06e27/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f74656e736f72666c6f775f666967757265372e706e67\" data-canonical-src=\"https://www.tensorflow.org/images/tensorflow_figure7.png\"\u003e\u003c/a\u003e\n\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIn \u003ca href=\"https://arxiv.org/abs/1604.00981\" rel=\"nofollow\"\u003ea recent paper\u003c/a\u003e, synchronous gradient\nupdates have demonstrated to reach higher accuracy in a shorter amount of time.\nIn this distributed Inception example we employ synchronous gradient updates.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNote that in this example each replica has a single tower that uses one GPU.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe command-line flags \u003ccode\u003eworker_hosts\u003c/code\u003e and \u003ccode\u003eps_hosts\u003c/code\u003e specify available servers.\nThe same binary will be used for both the \u003ccode\u003eworker\u003c/code\u003e jobs and the \u003ccode\u003eps\u003c/code\u003e jobs.\nCommand line flag \u003ccode\u003ejob_name\u003c/code\u003e will be used to specify what role a task will be\nplaying and \u003ccode\u003etask_id\u003c/code\u003e will be used to idenify which one of the jobs it is\nrunning. Several things to note here:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eThe numbers of \u003ccode\u003eps\u003c/code\u003e and \u003ccode\u003eworker\u003c/code\u003e tasks are inferred from the lists of hosts\nspecified in the flags. The \u003ccode\u003etask_id\u003c/code\u003e should be within the range \u003ccode\u003e[0, num_ps_tasks)\u003c/code\u003e for \u003ccode\u003eps\u003c/code\u003e tasks and \u003ccode\u003e[0, num_worker_tasks)\u003c/code\u003e for \u003ccode\u003eworker\u003c/code\u003e\ntasks.\u003c/li\u003e\n\u003cli\u003e\u003ccode\u003eps\u003c/code\u003e and \u003ccode\u003eworker\u003c/code\u003e tasks can run on the same machine, as long as that machine\nhas sufficient resources to handle both tasks. Note that the \u003ccode\u003eps\u003c/code\u003e task does\nnot benefit from a GPU, so it should not attempt to use one (see below).\u003c/li\u003e\n\u003cli\u003eMultiple \u003ccode\u003eworker\u003c/code\u003e tasks can run on the same machine with multiple GPUs so\nmachine_A with 2 GPUs may have 2 workers while machine_B with 1 GPU just has\n1 worker.\u003c/li\u003e\n\u003cli\u003eThe default learning rate schedule works well for a wide range of number of\nreplicas [25, 50, 100] but feel free to tune it for even better results.\u003c/li\u003e\n\u003cli\u003eThe command line of both \u003ccode\u003eps\u003c/code\u003e and \u003ccode\u003eworker\u003c/code\u003e tasks should include the complete\nlist of \u003ccode\u003eps_hosts\u003c/code\u003e and \u003ccode\u003eworker_hosts\u003c/code\u003e.\u003c/li\u003e\n\u003cli\u003eThere is a chief \u003ccode\u003eworker\u003c/code\u003e among all workers which defaults to \u003ccode\u003eworker\u003c/code\u003e 0.\nThe chief will be in charge of initializing all the parameters, writing out\nthe summaries and the checkpoint. The checkpoint and summary will be in the\n\u003ccode\u003etrain_dir\u003c/code\u003e of the host for \u003ccode\u003eworker\u003c/code\u003e 0.\u003c/li\u003e\n\u003cli\u003eEach worker processes a batch_size number of examples but each gradient\nupdate is computed from all replicas. Hence, the effective batch size of\nthis model is batch_size * num_workers.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/imagenet_distributed_train\n\n# To start worker 0, go to the worker0 host and run the following (Note that\n# task_id should be in the range [0, num_worker_tasks):\nbazel-bin/inception/imagenet_distributed_train \\\n--batch_size=32 \\\n--data_dir=$HOME/imagenet-data \\\n--job_name='worker' \\\n--task_id=0 \\\n--ps_hosts='ps0.example.com:2222' \\\n--worker_hosts='worker0.example.com:2222,worker1.example.com:2222'\n\n# To start worker 1, go to the worker1 host and run the following (Note that\n# task_id should be in the range [0, num_worker_tasks):\nbazel-bin/inception/imagenet_distributed_train \\\n--batch_size=32 \\\n--data_dir=$HOME/imagenet-data \\\n--job_name='worker' \\\n--task_id=1 \\\n--ps_hosts='ps0.example.com:2222' \\\n--worker_hosts='worker0.example.com:2222,worker1.example.com:2222'\n\n# To start the parameter server (ps), go to the ps host and run the following (Note\n# that task_id should be in the range [0, num_ps_tasks):\nbazel-bin/inception/imagenet_distributed_train \\\n--job_name='ps' \\\n--task_id=0 \\\n--ps_hosts='ps0.example.com:2222' \\\n--worker_hosts='worker0.example.com:2222,worker1.example.com:2222'\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/imagenet_distributed_train\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e To start worker 0, go to the worker0 host and run the following (Note that\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e task_id should be in the range [0, num_worker_tasks):\u003c/span\u003e\nbazel-bin/inception/imagenet_distributed_train \\\n--batch_size=32 \\\n--data_dir=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/imagenet-data \\\n--job_name=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--task_id=0 \\\n--ps_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--worker_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222,worker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e To start worker 1, go to the worker1 host and run the following (Note that\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e task_id should be in the range [0, num_worker_tasks):\u003c/span\u003e\nbazel-bin/inception/imagenet_distributed_train \\\n--batch_size=32 \\\n--data_dir=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/imagenet-data \\\n--job_name=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--task_id=1 \\\n--ps_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--worker_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222,worker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e To start the parameter server (ps), go to the ps host and run the following (Note\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e that task_id should be in the range [0, num_ps_tasks):\u003c/span\u003e\nbazel-bin/inception/imagenet_distributed_train \\\n--job_name=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--task_id=0 \\\n--ps_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--worker_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222,worker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIf you have installed a GPU-compatible version of TensorFlow, the \u003ccode\u003eps\u003c/code\u003e will also\ntry to allocate GPU memory although it is not helpful. This could potentially\ncrash the worker on the same machine as it has little to no GPU memory to\nallocate. To avoid this, you can prepend the previous command to start \u003ccode\u003eps\u003c/code\u003e\nwith: \u003ccode\u003eCUDA_VISIBLE_DEVICES=''\u003c/code\u003e\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"CUDA_VISIBLE_DEVICES='' bazel-bin/inception/imagenet_distributed_train \\\n--job_name='ps' \\\n--task_id=0 \\\n--ps_hosts='ps0.example.com:2222' \\\n--worker_hosts='worker0.example.com:2222,worker1.example.com:2222'\"\u003e\u003cpre\u003eCUDA_VISIBLE_DEVICES=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e bazel-bin/inception/imagenet_distributed_train \\\n--job_name=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--task_id=0 \\\n--ps_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e \\\n--worker_hosts=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222,worker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIf you have run everything correctly, you should see a log in each \u003ccode\u003eworker\u003c/code\u003e job\nthat looks like the following. Note the training speed varies depending on your\nhardware and the first several steps could take much longer.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222']\nINFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222']\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -\u0026gt; {ps0.example.com:2222, ps1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -\u0026gt; {localhost:2222, worker1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222\nINFO:tensorflow:Created variable global_step:0 with shape () and init \u0026lt;function zeros_initializer at 0x7f6aa014b140\u0026gt;\n\n...\n\nINFO:tensorflow:Created variable logits/logits/biases:0 with shape (1001,) and init \u0026lt;function _initializer at 0x7f6a77f3cf50\u0026gt;\nINFO:tensorflow:SyncReplicas enabled: replicas_to_aggregate=2; total_num_replicas=2\nINFO:tensorflow:2016-04-13 01:56:26.405639 Supervisor\nINFO:tensorflow:Started 2 queues for processing input data.\nINFO:tensorflow:global_step/sec: 0\nINFO:tensorflow:Worker 0: 2016-04-13 01:58:40.342404: step 0, loss = 12.97(0.0 examples/sec; 65.428 sec/batch)\nINFO:tensorflow:global_step/sec: 0.0172907\n...\"\u003e\u003cpre\u003eINFO:tensorflow:PS hosts are: [\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e]\nINFO:tensorflow:Worker hosts are: [\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e]\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e job ps -\u003cspan class=\"pl-k\"\u003e\u0026gt;\u003c/span\u003e {ps0.example.com:2222, ps1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e job worker -\u003cspan class=\"pl-k\"\u003e\u0026gt;\u003c/span\u003e {localhost:2222, worker1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222\nINFO:tensorflow:Created variable global_step:0 with \u003cspan class=\"pl-en\"\u003eshape\u003c/span\u003e () and init \u003cspan class=\"pl-k\"\u003e\u0026lt;\u003c/span\u003efunction zeros_initializer at 0x7f6aa014b\u003cspan class=\"pl-k\"\u003e140\u0026gt;\u003c/span\u003e\n\n...\n\nINFO:tensorflow:Created variable logits/logits/biases:0 with shape (1001,) and init \u003cspan class=\"pl-k\"\u003e\u0026lt;\u003c/span\u003efunction _initializer at 0x7f6a77f3cf\u003cspan class=\"pl-k\"\u003e50\u0026gt;\u003c/span\u003e\nINFO:tensorflow:SyncReplicas enabled: replicas_to_aggregate=2\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e total_num_replicas=2\nINFO:tensorflow:2016-04-13 01:56:26.405639 Supervisor\nINFO:tensorflow:Started 2 queues \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e processing input data.\nINFO:tensorflow:global_step/sec: 0\nINFO:tensorflow:Worker 0: 2016-04-13 01:58:40.342404: step 0, loss = 12.97(0.0 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 65.428 sec/batch)\nINFO:tensorflow:global_step/sec: 0.0172907\n...\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eand a log in each \u003ccode\u003eps\u003c/code\u003e job that looks like the following:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222']\nINFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222']\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -\u0026gt; {localhost:2222, ps1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -\u0026gt; {worker0.example.com:2222, worker1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222\"\u003e\u003cpre\u003eINFO:tensorflow:PS hosts are: [\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eps1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e]\nINFO:tensorflow:Worker hosts are: [\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker0.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e, \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003eworker1.example.com:2222\u003cspan class=\"pl-pds\"\u003e'\u003c/span\u003e\u003c/span\u003e]\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e job ps -\u003cspan class=\"pl-k\"\u003e\u0026gt;\u003c/span\u003e {localhost:2222, ps1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache \u003cspan class=\"pl-k\"\u003efor\u003c/span\u003e job worker -\u003cspan class=\"pl-k\"\u003e\u0026gt;\u003c/span\u003e {worker0.example.com:2222, worker1.example.com:2222}\nI tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.youtube.com/watch?v=9bZkp7q19f0\" rel=\"nofollow\"\u003eCongratulations!\u003c/a\u003e You are now\ntraining Inception in a distributed manner.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Evaluate\u003c/h2\u003e\u003ca id=\"user-content-how-to-evaluate\" class=\"anchor\" aria-label=\"Permalink: How to Evaluate\" href=\"#how-to-evaluate\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eEvaluating an Inception v3 model on the ImageNet 2012 validation data set\nrequires running a separate binary.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe evaluation procedure is nearly identical to [Evaluating a Model]\n(\u003ca href=\"https://www.tensorflow.org/tutorials/deep_cnn/index.html#evaluating-a-model\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/tutorials/deep_cnn/index.html#evaluating-a-model\u003c/a\u003e)\ndescribed in the [Convolutional Neural Network]\n(\u003ca href=\"https://www.tensorflow.org/tutorials/deep_cnn/index.html\" rel=\"nofollow\"\u003ehttps://www.tensorflow.org/tutorials/deep_cnn/index.html\u003c/a\u003e) tutorial.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eWARNING\u003c/strong\u003e Be careful not to run the evaluation and training binary on the same\nGPU or else you might run out of memory. Consider running the evaluation on a\nseparate GPU if available or suspending the training binary while running the\nevaluation on the same GPU.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eBriefly, one can evaluate the model by running:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/imagenet_eval\n\n# run it\nbazel-bin/inception/imagenet_eval --checkpoint_dir=/tmp/imagenet_train --eval_dir=/tmp/imagenet_eval\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/imagenet_eval\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e run it\u003c/span\u003e\nbazel-bin/inception/imagenet_eval --checkpoint_dir=/tmp/imagenet_train --eval_dir=/tmp/imagenet_eval\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eNote that we point \u003ccode\u003e--checkpoint_dir\u003c/code\u003e to the location of the checkpoints saved\nby \u003ccode\u003einception_train.py\u003c/code\u003e above. Running the above command results in the\nfollowing output:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"2016-02-17 22:32:50.391206: precision @ 1 = 0.735\n...\"\u003e\u003cpre\u003e2016-02-17 22:32:50.391206: precision @ 1 = 0.735\n...\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe script calculates the precision @ 1 over the entire validation data\nperiodically. The precision @ 1 measures the how often the highest scoring\nprediction from the model matched the ImageNet label -- in this case, 73.5%. If\nyou wish to run the eval just once and not periodically, append the \u003ccode\u003e--run_once\u003c/code\u003e\noption.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eMuch like the training script, \u003ccode\u003eimagenet_eval.py\u003c/code\u003e also exports summaries that\nmay be visualized in TensorBoard. These summaries calculate additional\nstatistics on the predictions (e.g. recall @ 5) as well as monitor the\nstatistics of the model activations and weights during evaluation.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Fine-Tune a Pre-Trained Model on a New Task\u003c/h2\u003e\u003ca id=\"user-content-how-to-fine-tune-a-pre-trained-model-on-a-new-task\" class=\"anchor\" aria-label=\"Permalink: How to Fine-Tune a Pre-Trained Model on a New Task\" href=\"#how-to-fine-tune-a-pre-trained-model-on-a-new-task\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eGetting Started\u003c/h3\u003e\u003ca id=\"user-content-getting-started-1\" class=\"anchor\" aria-label=\"Permalink: Getting Started\" href=\"#getting-started-1\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eMuch like training the ImageNet model we must first convert a new data set to\nthe sharded TFRecord format which each entry is a serialized \u003ccode\u003etf.Example\u003c/code\u003e proto.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eWe have provided a script demonstrating how to do this for small data set of of\na few thousand flower images spread across 5 labels:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"daisy, dandelion, roses, sunflowers, tulips\"\u003e\u003cpre\u003edaisy, dandelion, roses, sunflowers, tulips\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThere is a single automated script that downloads the data set and converts it\nto the TFRecord format. Much like the ImageNet data set, each record in the\nTFRecord format is a serialized \u003ccode\u003etf.Example\u003c/code\u003e proto whose entries include a\nJPEG-encoded string and an integer label. Please see [\u003ccode\u003eparse_example_proto\u003c/code\u003e]\n(inception/image_processing.py) for details.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe script just takes a few minutes to run depending your network connection\nspeed for downloading and processing the images. Your hard disk requires 200MB\nof free storage. Here we select \u003ccode\u003eDATA_DIR=$HOME/flowers-data\u003c/code\u003e as such a location\nbut feel free to edit accordingly.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# location of where to place the flowers data\nFLOWERS_DATA_DIR=$HOME/flowers-data\n\n# build the preprocessing script.\nbazel build inception/download_and_preprocess_flowers\n\n# run it\nbazel-bin/inception/download_and_preprocess_flowers \u0026quot;${FLOWERS_DATA_DIR}\u0026quot;\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e location of where to place the flowers data\u003c/span\u003e\nFLOWERS_DATA_DIR=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/flowers-data\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e build the preprocessing script.\u003c/span\u003e\nbazel build inception/download_and_preprocess_flowers\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e run it\u003c/span\u003e\nbazel-bin/inception/download_and_preprocess_flowers \u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${FLOWERS_DATA_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIf the script runs successfully, the final line of the terminal output should\nlook like:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"2016-02-24 20:42:25.067551: Finished writing all 3170 images in data set.\"\u003e\u003cpre\u003e2016-02-24 20:42:25.067551: Finished writing all 3170 images \u003cspan class=\"pl-k\"\u003ein\u003c/span\u003e data set.\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWhen the script finishes you will find 2 shards for the training and validation\nfiles in the \u003ccode\u003eDATA_DIR\u003c/code\u003e. The files will match the patterns \u003ccode\u003etrain-????-of-00001\u003c/code\u003e\nand \u003ccode\u003evalidation-?????-of-00001\u003c/code\u003e, respectively.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eNOTE\u003c/strong\u003e If you wish to prepare a custom image data set for transfer learning,\nyou will need to invoke \u003ca href=\"/wenwei202/terngrad/blob/master/inception/data/build_image_data.py\"\u003e\u003ccode\u003ebuild_image_data.py\u003c/code\u003e\u003c/a\u003e on\nyour custom data set. Please see the associated options and assumptions behind\nthis script by reading the comments section of [\u003ccode\u003ebuild_image_data.py\u003c/code\u003e]\n(inception/data/build_image_data.py). Also, if your custom data has a different\nnumber of examples or classes, you need to change the appropriate values in\n\u003ca href=\"/wenwei202/terngrad/blob/master/inception/imagenet_data.py\"\u003e\u003ccode\u003eimagenet_data.py\u003c/code\u003e\u003c/a\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe second piece you will need is a trained Inception v3 image model. You have\nthe option of either training one yourself (See [How to Train from Scratch]\n(#how-to-train-from-scratch) for details) or you can download a pre-trained\nmodel like so:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# location of where to place the Inception v3 model\nDATA_DIR=$HOME/inception-v3-model\ncd ${DATA_DIR}\n\n# download the Inception v3 model\ncurl -O http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz\ntar xzf inception-v3-2016-03-01.tar.gz\n\n# this will create a directory called inception-v3 which contains the following files.\n\u0026gt; ls inception-v3\nREADME.txt\ncheckpoint\nmodel.ckpt-157585\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e location of where to place the Inception v3 model\u003c/span\u003e\nDATA_DIR=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/inception-v3-model\n\u003cspan class=\"pl-c1\"\u003ecd\u003c/span\u003e \u003cspan class=\"pl-smi\"\u003e${DATA_DIR}\u003c/span\u003e\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e download the Inception v3 model\u003c/span\u003e\ncurl -O http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz\ntar xzf inception-v3-2016-03-01.tar.gz\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e this will create a directory called inception-v3 which contains the following files.\u003c/span\u003e\n\u003cspan class=\"pl-k\"\u003e\u0026gt;\u003c/span\u003e ls inception-v3\nREADME.txt\ncheckpoint\nmodel.ckpt-157585\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003e\u003ca href=\"https://www.youtube.com/watch?v=9bZkp7q19f0\" rel=\"nofollow\"\u003eCongratulations!\u003c/a\u003e You are now\nready to fine-tune your pre-trained Inception v3 model with the flower data set.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Retrain a Trained Model on the Flowers Data\u003c/h3\u003e\u003ca id=\"user-content-how-to-retrain-a-trained-model-on-the-flowers-data\" class=\"anchor\" aria-label=\"Permalink: How to Retrain a Trained Model on the Flowers Data\" href=\"#how-to-retrain-a-trained-model-on-the-flowers-data\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe are now ready to fine-tune a pre-trained Inception-v3 model on the flowers\ndata set. This requires two distinct changes to our training procedure:\u003c/p\u003e\n\u003col dir=\"auto\"\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eBuild the exact same model as previously except we change the number of\nlabels in the final classification layer.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eRestore all weights from the pre-trained Inception-v3 except for the final\nclassification layer; this will get randomly initialized instead.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp dir=\"auto\"\u003eWe can perform these two operations by specifying two flags:\n\u003ccode\u003e--pretrained_model_checkpoint_path\u003c/code\u003e and \u003ccode\u003e--fine_tune\u003c/code\u003e. The first flag is a\nstring that points to the path of a pre-trained Inception-v3 model. If this flag\nis specified, it will load the entire model from the checkpoint before the\nscript begins training.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe second flag \u003ccode\u003e--fine_tune\u003c/code\u003e is a boolean that indicates whether the last\nclassification layer should be randomly initialized or restored. You may set\nthis flag to false if you wish to continue training a pre-trained model from a\ncheckpoint. If you set this flag to true, you can train a new classification\nlayer from scratch.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIn order to understand how \u003ccode\u003e--fine_tune\u003c/code\u003e works, please see the discussion on\n\u003ccode\u003eVariables\u003c/code\u003e in the TensorFlow-Slim \u003ca href=\"/wenwei202/terngrad/blob/master/inception/slim/README.md\"\u003e\u003ccode\u003eREADME.md\u003c/code\u003e\u003c/a\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003ePutting this all together you can retrain a pre-trained Inception-v3 model on\nthe flowers data set with the following command.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/flowers_train\n\n# Path to the downloaded Inception-v3 model.\nMODEL_PATH=\u0026quot;${INCEPTION_MODEL_DIR}/model.ckpt-157585\u0026quot;\n\n# Directory where the flowers data resides.\nFLOWERS_DATA_DIR=/tmp/flowers-data/\n\n# Directory where to save the checkpoint and events files.\nTRAIN_DIR=/tmp/flowers_train/\n\n# Run the fine-tuning on the flowers data set starting from the pre-trained\n# Imagenet-v3 model.\nbazel-bin/inception/flowers_train \\\n --train_dir=\u0026quot;${TRAIN_DIR}\u0026quot; \\\n --data_dir=\u0026quot;${FLOWERS_DATA_DIR}\u0026quot; \\\n --pretrained_model_checkpoint_path=\u0026quot;${MODEL_PATH}\u0026quot; \\\n --fine_tune=True \\\n --initial_learning_rate=0.001 \\\n --input_queue_memory_factor=1\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/flowers_train\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Path to the downloaded Inception-v3 model.\u003c/span\u003e\nMODEL_PATH=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${INCEPTION_MODEL_DIR}\u003c/span\u003e/model.ckpt-157585\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Directory where the flowers data resides.\u003c/span\u003e\nFLOWERS_DATA_DIR=/tmp/flowers-data/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Directory where to save the checkpoint and events files.\u003c/span\u003e\nTRAIN_DIR=/tmp/flowers_train/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Run the fine-tuning on the flowers data set starting from the pre-trained\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Imagenet-v3 model.\u003c/span\u003e\nbazel-bin/inception/flowers_train \\\n --train_dir=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${TRAIN_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --data_dir=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${FLOWERS_DATA_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --pretrained_model_checkpoint_path=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${MODEL_PATH}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --fine_tune=True \\\n --initial_learning_rate=0.001 \\\n --input_queue_memory_factor=1\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe have added a few extra options to the training procedure.\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eFine-tuning a model a separate data set requires significantly lowering the\ninitial learning rate. We set the initial learning rate to 0.001.\u003c/li\u003e\n\u003cli\u003eThe flowers data set is quite small so we shrink the size of the shuffling\nqueue of examples. See \u003ca href=\"#adjusting-memory-demands\"\u003eAdjusting Memory Demands\u003c/a\u003e\nfor more details.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eThe training script will only reports the loss. To evaluate the quality of the\nfine-tuned model, you will need to run \u003ccode\u003eflowers_eval\u003c/code\u003e:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# Build the model. Note that we need to make sure the TensorFlow is ready to\n# use before this as this command will not build TensorFlow.\nbazel build inception/flowers_eval\n\n# Directory where we saved the fine-tuned checkpoint and events files.\nTRAIN_DIR=/tmp/flowers_train/\n\n# Directory where the flowers data resides.\nFLOWERS_DATA_DIR=/tmp/flowers-data/\n\n# Directory where to save the evaluation events files.\nEVAL_DIR=/tmp/flowers_eval/\n\n# Evaluate the fine-tuned model on a hold-out of the flower data set.\nbazel-bin/inception/flowers_eval \\\n --eval_dir=\u0026quot;${EVAL_DIR}\u0026quot; \\\n --data_dir=\u0026quot;${FLOWERS_DATA_DIR}\u0026quot; \\\n --subset=validation \\\n --num_examples=500 \\\n --checkpoint_dir=\u0026quot;${TRAIN_DIR}\u0026quot; \\\n --input_queue_memory_factor=1 \\\n --run_once\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Build the model. Note that we need to make sure the TensorFlow is ready to\u003c/span\u003e\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e use before this as this command will not build TensorFlow.\u003c/span\u003e\nbazel build inception/flowers_eval\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Directory where we saved the fine-tuned checkpoint and events files.\u003c/span\u003e\nTRAIN_DIR=/tmp/flowers_train/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Directory where the flowers data resides.\u003c/span\u003e\nFLOWERS_DATA_DIR=/tmp/flowers-data/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Directory where to save the evaluation events files.\u003c/span\u003e\nEVAL_DIR=/tmp/flowers_eval/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Evaluate the fine-tuned model on a hold-out of the flower data set.\u003c/span\u003e\nbazel-bin/inception/flowers_eval \\\n --eval_dir=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${EVAL_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --data_dir=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${FLOWERS_DATA_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --subset=validation \\\n --num_examples=500 \\\n --checkpoint_dir=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${TRAIN_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --input_queue_memory_factor=1 \\\n --run_once\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe find that the evaluation arrives at roughly 93.4% precision@1 after the model\nhas been running for 2000 steps.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"Succesfully loaded model from /tmp/flowers/model.ckpt-1999 at step=1999.\n2016-03-01 16:52:51.761219: starting evaluation on (validation).\n2016-03-01 16:53:05.450419: [20 batches out of 20] (36.5 examples/sec; 0.684sec/batch)\n2016-03-01 16:53:05.450471: precision @ 1 = 0.9340 recall @ 5 = 0.9960 [500 examples]\"\u003e\u003cpre\u003eSuccesfully loaded model from /tmp/flowers/model.ckpt-1999 at step=1999.\n2016-03-01 16:52:51.761219: starting evaluation on (validation).\n2016-03-01 16:53:05.450419: [20 batches out of 20] (36.5 examples/sec\u003cspan class=\"pl-k\"\u003e;\u003c/span\u003e 0.684sec/batch)\n2016-03-01 16:53:05.450471: precision @ 1 = 0.9340 recall @ 5 = 0.9960 [500 examples]\u003c/pre\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow to Construct a New Dataset for Retraining\u003c/h2\u003e\u003ca id=\"user-content-how-to-construct-a-new-dataset-for-retraining\" class=\"anchor\" aria-label=\"Permalink: How to Construct a New Dataset for Retraining\" href=\"#how-to-construct-a-new-dataset-for-retraining\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eOne can use the existing scripts supplied with this model to build a new dataset\nfor training or fine-tuning. The main script to employ is\n\u003ca href=\"/wenwei202/terngrad/blob/master/inception/data/build_image_data.py\"\u003e\u003ccode\u003ebuild_image_data.py\u003c/code\u003e\u003c/a\u003e. Briefly, this script takes a\nstructured directory of images and converts it to a sharded \u003ccode\u003eTFRecord\u003c/code\u003e that can\nbe read by the Inception model.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIn particular, you will need to create a directory of training images that\nreside within \u003ccode\u003e$TRAIN_DIR\u003c/code\u003e and \u003ccode\u003e$VALIDATION_DIR\u003c/code\u003e arranged as such:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\" $TRAIN_DIR/dog/image0.jpeg\n $TRAIN_DIR/dog/image1.jpg\n $TRAIN_DIR/dog/image2.png\n ...\n $TRAIN_DIR/cat/weird-image.jpeg\n $TRAIN_DIR/cat/my-image.jpeg\n $TRAIN_DIR/cat/my-image.JPG\n ...\n $VALIDATION_DIR/dog/imageA.jpeg\n $VALIDATION_DIR/dog/imageB.jpg\n $VALIDATION_DIR/dog/imageC.png\n ...\n $VALIDATION_DIR/cat/weird-image.PNG\n $VALIDATION_DIR/cat/that-image.jpg\n $VALIDATION_DIR/cat/cat.JPG\n ...\"\u003e\u003cpre\u003e \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/dog/image0.jpeg\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/dog/image1.jpg\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/dog/image2.png\n ...\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/cat/weird-image.jpeg\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/cat/my-image.jpeg\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/cat/my-image.JPG\n ...\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/dog/imageA.jpeg\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/dog/imageB.jpg\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/dog/imageC.png\n ...\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/cat/weird-image.PNG\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/cat/that-image.jpg\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/cat/cat.JPG\n ...\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eEach sub-directory in \u003ccode\u003e$TRAIN_DIR\u003c/code\u003e and \u003ccode\u003e$VALIDATION_DIR\u003c/code\u003e corresponds to a unique\nlabel for the images that reside within that sub-directory. The images may be\nJPEG or PNG images. We do not support other images types currently.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eOnce the data is arranged in this directory structure, we can run\n\u003ccode\u003ebuild_image_data.py\u003c/code\u003e on the data to generate the sharded \u003ccode\u003eTFRecord\u003c/code\u003e dataset.\nEach entry of the \u003ccode\u003eTFRecord\u003c/code\u003e is a serialized \u003ccode\u003etf.Example\u003c/code\u003e protocol buffer. A\ncomplete list of information contained in the \u003ccode\u003etf.Example\u003c/code\u003e is described in the\ncomments of \u003ccode\u003ebuild_image_data.py\u003c/code\u003e.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eTo run \u003ccode\u003ebuild_image_data.py\u003c/code\u003e, you can run the following command line:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"# location to where to save the TFRecord data.\nOUTPUT_DIRECTORY=$HOME/my-custom-data/\n\n# build the preprocessing script.\nbazel build inception/build_image_data\n\n# convert the data.\nbazel-bin/inception/build_image_data \\\n --train_directory=\u0026quot;${TRAIN_DIR}\u0026quot; \\\n --validation_directory=\u0026quot;${VALIDATION_DIR}\u0026quot; \\\n --output_directory=\u0026quot;${OUTPUT_DIRECTORY}\u0026quot; \\\n --labels_file=\u0026quot;${LABELS_FILE}\u0026quot; \\\n --train_shards=128 \\\n --validation_shards=24 \\\n --num_threads=8\"\u003e\u003cpre\u003e\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e location to where to save the TFRecord data.\u003c/span\u003e\nOUTPUT_DIRECTORY=\u003cspan class=\"pl-smi\"\u003e$HOME\u003c/span\u003e/my-custom-data/\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e build the preprocessing script.\u003c/span\u003e\nbazel build inception/build_image_data\n\n\u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e convert the data.\u003c/span\u003e\nbazel-bin/inception/build_image_data \\\n --train_directory=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${TRAIN_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --validation_directory=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${VALIDATION_DIR}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --output_directory=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${OUTPUT_DIRECTORY}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --labels_file=\u003cspan class=\"pl-s\"\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003cspan class=\"pl-smi\"\u003e${LABELS_FILE}\u003c/span\u003e\u003cspan class=\"pl-pds\"\u003e\"\u003c/span\u003e\u003c/span\u003e \\\n --train_shards=128 \\\n --validation_shards=24 \\\n --num_threads=8\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003ewhere the \u003ccode\u003e$OUTPUT_DIRECTORY\u003c/code\u003e is the location of the sharded \u003ccode\u003eTFRecords\u003c/code\u003e. The\n\u003ccode\u003e$LABELS_FILE\u003c/code\u003e will be a text file that is read by the script that provides\na list of all of the labels. For instance, in the case flowers data set, the\n\u003ccode\u003e$LABELS_FILE\u003c/code\u003e contained the following data:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"daisy\ndandelion\nroses\nsunflowers\ntulips\"\u003e\u003cpre\u003edaisy\ndandelion\nroses\nsunflowers\ntulips\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eNote that each row of each label corresponds with the entry in the final\nclassifier in the model. That is, the \u003ccode\u003edaisy\u003c/code\u003e corresponds to the classifier for\nentry \u003ccode\u003e1\u003c/code\u003e; \u003ccode\u003edandelion\u003c/code\u003e is entry \u003ccode\u003e2\u003c/code\u003e, etc. We skip label \u003ccode\u003e0\u003c/code\u003e as a background\nclass.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eAfter running this script produces files that look like the following:\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\" $TRAIN_DIR/train-00000-of-00024\n $TRAIN_DIR/train-00001-of-00024\n ...\n $TRAIN_DIR/train-00023-of-00024\n\nand\n\n $VALIDATION_DIR/validation-00000-of-00008\n $VALIDATION_DIR/validation-00001-of-00008\n ...\n $VALIDATION_DIR/validation-00007-of-00008\"\u003e\u003cpre\u003e \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/train-00000-of-00024\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/train-00001-of-00024\n ...\n \u003cspan class=\"pl-smi\"\u003e$TRAIN_DIR\u003c/span\u003e/train-00023-of-00024\n\nand\n\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/validation-00000-of-00008\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/validation-00001-of-00008\n ...\n \u003cspan class=\"pl-smi\"\u003e$VALIDATION_DIR\u003c/span\u003e/validation-00007-of-00008\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003ewhere 24 and 8 are the number of shards specified for each dataset,\nrespectively. Generally speaking, we aim for selecting the number of shards such\nthat roughly 1024 images reside in each shard. Once this data set is built, you\nare ready to train or fine-tune an Inception model on this data set.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eNote, if you are piggy backing on the flowers retraining scripts, be sure to\nupdate \u003ccode\u003enum_classes()\u003c/code\u003e and \u003ccode\u003enum_examples_per_epoch()\u003c/code\u003e in \u003ccode\u003eflowers_data.py\u003c/code\u003e\nto correspond with your data.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003ePractical Considerations for Training a Model\u003c/h2\u003e\u003ca id=\"user-content-practical-considerations-for-training-a-model\" class=\"anchor\" aria-label=\"Permalink: Practical Considerations for Training a Model\" href=\"#practical-considerations-for-training-a-model\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe model architecture and training procedure is heavily dependent on the\nhardware used to train the model. If you wish to train or fine-tune this model\non your machine \u003cstrong\u003eyou will need to adjust and empirically determine a good set\nof training hyper-parameters for your setup\u003c/strong\u003e. What follows are some general\nconsiderations for novices.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eFinding Good Hyperparameters\u003c/h3\u003e\u003ca id=\"user-content-finding-good-hyperparameters\" class=\"anchor\" aria-label=\"Permalink: Finding Good Hyperparameters\" href=\"#finding-good-hyperparameters\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eRoughly 5-10 hyper-parameters govern the speed at which a network is trained. In\naddition to \u003ccode\u003e--batch_size\u003c/code\u003e and \u003ccode\u003e--num_gpus\u003c/code\u003e, there are several constants defined\nin \u003ca href=\"/wenwei202/terngrad/blob/master/inception/inception_train.py\"\u003einception_train.py\u003c/a\u003e which dictate the learning\nschedule.\u003c/p\u003e\n\u003cdiv class=\"highlight highlight-source-shell notranslate position-relative overflow-auto\" dir=\"auto\" data-snippet-clipboard-copy-content=\"RMSPROP_DECAY = 0.9 # Decay term for RMSProp.\nMOMENTUM = 0.9 # Momentum in RMSProp.\nRMSPROP_EPSILON = 1.0 # Epsilon term for RMSProp.\nINITIAL_LEARNING_RATE = 0.1 # Initial learning rate.\nNUM_EPOCHS_PER_DECAY = 30.0 # Epochs after which learning rate decays.\nLEARNING_RATE_DECAY_FACTOR = 0.16 # Learning rate decay factor.\"\u003e\u003cpre\u003eRMSPROP_DECAY = 0.9 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Decay term for RMSProp.\u003c/span\u003e\nMOMENTUM = 0.9 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Momentum in RMSProp.\u003c/span\u003e\nRMSPROP_EPSILON = 1.0 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Epsilon term for RMSProp.\u003c/span\u003e\nINITIAL_LEARNING_RATE = 0.1 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Initial learning rate.\u003c/span\u003e\nNUM_EPOCHS_PER_DECAY = 30.0 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Epochs after which learning rate decays.\u003c/span\u003e\nLEARNING_RATE_DECAY_FACTOR = 0.16 \u003cspan class=\"pl-c\"\u003e\u003cspan class=\"pl-c\"\u003e#\u003c/span\u003e Learning rate decay factor.\u003c/span\u003e\u003c/pre\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThere are many papers that discuss the various tricks and trade-offs associated\nwith training a model with stochastic gradient descent. For those new to the\nfield, some great references are:\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003eY Bengio, \u003ca href=\"http://arxiv.org/abs/1206.5533\" rel=\"nofollow\"\u003ePractical recommendations for gradient-based training of deep\narchitectures\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eI Goodfellow, Y Bengio and A Courville, [Deep Learning]\n(\u003ca href=\"http://www.deeplearningbook.org/\" rel=\"nofollow\"\u003ehttp://www.deeplearningbook.org/\u003c/a\u003e)\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003eWhat follows is a summary of some general advice for identifying appropriate\nmodel hyper-parameters in the context of this particular model training setup.\nNamely, this library provides \u003cem\u003esynchronous\u003c/em\u003e updates to model parameters based on\nbatch-splitting the model across multiple GPUs.\u003c/p\u003e\n\u003cul dir=\"auto\"\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eHigher learning rates leads to faster training. Too high of learning rate\nleads to instability and will cause model parameters to diverge to infinity\nor NaN.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eLarger batch sizes lead to higher quality estimates of the gradient and\npermit training the model with higher learning rates.\u003c/p\u003e\n\u003c/li\u003e\n\u003cli\u003e\n\u003cp dir=\"auto\"\u003eOften the GPU memory is a bottleneck that prevents employing larger batch\nsizes. Employing more GPUs allows one to user larger batch sizes because\nthis model splits the batch across the GPUs.\u003c/p\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp dir=\"auto\"\u003e\u003cstrong\u003eNOTE\u003c/strong\u003e If one wishes to train this model with \u003cem\u003easynchronous\u003c/em\u003e gradient updates,\none will need to substantially alter this model and new considerations need to\nbe factored into hyperparameter tuning. See \u003ca href=\"http://research.google.com/archive/large_deep_networks_nips2012.html\" rel=\"nofollow\"\u003eLarge Scale Distributed Deep\nNetworks\u003c/a\u003e\nfor a discussion in this domain.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch3 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eAdjusting Memory Demands\u003c/h3\u003e\u003ca id=\"user-content-adjusting-memory-demands\" class=\"anchor\" aria-label=\"Permalink: Adjusting Memory Demands\" href=\"#adjusting-memory-demands\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eTraining this model has large memory demands in terms of the CPU and GPU. Let's\ndiscuss each item in turn.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eGPU memory is relatively small compared to CPU memory. Two items dictate the\namount of GPU memory employed -- model architecture and batch size. Assuming\nthat you keep the model architecture fixed, the sole parameter governing the GPU\ndemand is the batch size. A good rule of thumb is to try employ as large of\nbatch size as will fit on the GPU.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIf you run out of GPU memory, either lower the \u003ccode\u003e--batch_size\u003c/code\u003e or employ more\nGPUs on your desktop. The model performs batch-splitting across GPUs, thus N\nGPUs can handle N times the batch size of 1 GPU.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eThe model requires a large amount of CPU memory as well. We have tuned the model\nto employ about ~20GB of CPU memory. Thus, having access to about 40 GB of CPU\nmemory would be ideal.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIf that is not possible, you can tune down the memory demands of the model via\nlowering \u003ccode\u003e--input_queue_memory_factor\u003c/code\u003e. Images are preprocessed asynchronously\nwith respect to the main training across \u003ccode\u003e--num_preprocess_threads\u003c/code\u003e threads. The\npreprocessed images are stored in shuffling queue in which each GPU performs a\ndequeue operation in order to receive a \u003ccode\u003ebatch_size\u003c/code\u003e worth of images.\u003c/p\u003e\n\u003cp dir=\"auto\"\u003eIn order to guarantee good shuffling across the data, we maintain a large\nshuffling queue of 1024 x \u003ccode\u003einput_queue_memory_factor\u003c/code\u003e images. For the current\nmodel architecture, this corresponds to about 4GB of CPU memory. You may lower\n\u003ccode\u003einput_queue_memory_factor\u003c/code\u003e in order to decrease the memory footprint. Keep in\nmind though that lowering this value drastically may result in a model with\nslightly lower predictive accuracy when training from scratch. Please see\ncomments in \u003ca href=\"/wenwei202/terngrad/blob/master/inception/image_processing.py\"\u003e\u003ccode\u003eimage_processing.py\u003c/code\u003e\u003c/a\u003e for more details.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch2 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eTroubleshooting\u003c/h2\u003e\u003ca id=\"user-content-troubleshooting\" class=\"anchor\" aria-label=\"Permalink: Troubleshooting\" href=\"#troubleshooting\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe model runs out of CPU memory.\u003c/h4\u003e\u003ca id=\"user-content-the-model-runs-out-of-cpu-memory\" class=\"anchor\" aria-label=\"Permalink: The model runs out of CPU memory.\" href=\"#the-model-runs-out-of-cpu-memory\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eIn lieu of buying more CPU memory, an easy fix is to decrease\n\u003ccode\u003e--input_queue_memory_factor\u003c/code\u003e. See [Adjusting Memory Demands]\n(#adjusting-memory-demands).\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe model runs out of GPU memory.\u003c/h4\u003e\u003ca id=\"user-content-the-model-runs-out-of-gpu-memory\" class=\"anchor\" aria-label=\"Permalink: The model runs out of GPU memory.\" href=\"#the-model-runs-out-of-gpu-memory\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe data is not able to fit on the GPU card. The simplest solution is to\ndecrease the batch size of the model. Otherwise, you will need to think about a\nmore sophisticated method for specifying the training which cuts up the model\nacross multiple \u003ccode\u003esession.run()\u003c/code\u003e calls or partitions the model across multiple\nGPUs. See \u003ca href=\"https://www.tensorflow.org/how_tos/using_gpu/index.html\" rel=\"nofollow\"\u003eUsing GPUs\u003c/a\u003e\nand \u003ca href=\"#adjusting-memory-demands\"\u003eAdjusting Memory Demands\u003c/a\u003e for more information.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eThe model training results in NaN's.\u003c/h4\u003e\u003ca id=\"user-content-the-model-training-results-in-nans\" class=\"anchor\" aria-label=\"Permalink: The model training results in NaN's.\" href=\"#the-model-training-results-in-nans\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe learning rate of the model is too high. Turn down your learning rate.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eI wish to train a model with a different image size.\u003c/h4\u003e\u003ca id=\"user-content-i-wish-to-train-a-model-with-a-different-image-size\" class=\"anchor\" aria-label=\"Permalink: I wish to train a model with a different image size.\" href=\"#i-wish-to-train-a-model-with-a-different-image-size\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eThe simplest solution is to artificially resize your images to \u003ccode\u003e299x299\u003c/code\u003e pixels.\nSee \u003ca href=\"https://www.tensorflow.org/api_docs/python/image.html\" rel=\"nofollow\"\u003eImages\u003c/a\u003e section for\nmany resizing, cropping and padding methods. Note that the entire model\narchitecture is predicated on a \u003ccode\u003e299x299\u003c/code\u003e image, thus if you wish to change the\ninput image size, then you may need to redesign the entire model architecture.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eWhat hardware specification are these hyper-parameters targeted for?\u003c/h4\u003e\u003ca id=\"user-content-what-hardware-specification-are-these-hyper-parameters-targeted-for\" class=\"anchor\" aria-label=\"Permalink: What hardware specification are these hyper-parameters targeted for?\" href=\"#what-hardware-specification-are-these-hyper-parameters-targeted-for\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eWe targeted a desktop with 128GB of CPU ram connected to 8 NVIDIA Tesla K40 GPU\ncards but we have run this on desktops with 32GB of CPU ram and 1 NVIDIA Tesla\nK40. You can get a sense of the various training configurations we tested by\nreading the comments in \u003ca href=\"/wenwei202/terngrad/blob/master/inception/inception_train.py\"\u003e\u003ccode\u003einception_train.py\u003c/code\u003e\u003c/a\u003e.\u003c/p\u003e\n\u003cdiv class=\"markdown-heading\" dir=\"auto\"\u003e\u003ch4 tabindex=\"-1\" class=\"heading-element\" dir=\"auto\"\u003eHow do I continue training from a checkpoint in distributed setting?\u003c/h4\u003e\u003ca id=\"user-content-how-do-i-continue-training-from-a-checkpoint-in-distributed-setting\" class=\"anchor\" aria-label=\"Permalink: How do I continue training from a checkpoint in distributed setting?\" href=\"#how-do-i-continue-training-from-a-checkpoint-in-distributed-setting\"\u003e\u003csvg class=\"octicon octicon-link\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" height=\"16\" aria-hidden=\"true\"\u003e\u003cpath d=\"m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z\"\u003e\u003c/path\u003e\u003c/svg\u003e\u003c/a\u003e\u003c/div\u003e\n\u003cp dir=\"auto\"\u003eYou only need to make sure that the checkpoint is in a location that can be\nreached by all of the \u003ccode\u003eps\u003c/code\u003e tasks. By specifying the checkpoint location with\n\u003ccode\u003e--train_dir\u003c/code\u003e , the \u003ccode\u003eps\u003c/code\u003e servers will load the checkpoint before commencing\ntraining.\u003c/p\u003e\n\u003c/article\u003e","loaded":true,"timedOut":false,"errorMessage":null,"headerInfo":{"toc":[{"level":1,"text":"Introduction","anchor":"introduction","htmlText":"Introduction"},{"level":1,"text":"Dependencies","anchor":"dependencies","htmlText":"Dependencies"},{"level":1,"text":"Build all","anchor":"build-all","htmlText":"Build all"},{"level":1,"text":"Dataset preparation","anchor":"dataset-preparation","htmlText":"Dataset preparation"},{"level":2,"text":"Download and generate mnist TFRecord","anchor":"download-and-generate-mnist-tfrecord","htmlText":"Download and generate mnist TFRecord"},{"level":2,"text":"Download and generate cifar-10 TFRecord","anchor":"download-and-generate-cifar-10-tfrecord","htmlText":"Download and generate cifar-10 TFRecord"},{"level":2,"text":"Download and generate ImageNet TFRecord","anchor":"download-and-generate-imagenet-tfrecord","htmlText":"Download and generate ImageNet TFRecord"},{"level":1,"text":"Examples on multi-gpu mode","anchor":"examples-on-multi-gpu-mode","htmlText":"Examples on multi-gpu mode"},{"level":2,"text":"Training CifarNet by TernGrad with Adam","anchor":"training-cifarnet-by-terngrad-with-adam","htmlText":"Training CifarNet by TernGrad with Adam"},{"level":1,"text":"Examples on distributed-node mode","anchor":"examples-on-distributed-node-mode","htmlText":"Examples on distributed-node mode"},{"level":2,"text":"ssh setup","anchor":"ssh-setup","htmlText":"ssh setup"},{"level":2,"text":"A toy example","anchor":"a-toy-example","htmlText":"A toy example"},{"level":2,"text":"A single script to launch all","anchor":"a-single-script-to-launch-all","htmlText":"A single script to launch all"},{"level":1,"text":"Python executables","anchor":"python-executables","htmlText":"Python executables"},{"level":2,"text":"Build and run evaluating/training LeNet on mnist","anchor":"build-and-run-evaluatingtraining-lenet-on-mnist","htmlText":"Build and run evaluating/training LeNet on mnist"},{"level":2,"text":"Build and run evaluating/training on cifar-10","anchor":"build-and-run-evaluatingtraining-on-cifar-10","htmlText":"Build and run evaluating/training on cifar-10"},{"level":2,"text":"Build and run ImageNet","anchor":"build-and-run-imagenet","htmlText":"Build and run ImageNet"},{"level":1,"text":"Open questions","anchor":"open-questions","htmlText":"Open questions"},{"level":1,"text":"Backup (Inception in TensorFlow)","anchor":"backup-inception-in-tensorflow","htmlText":"Backup (Inception in TensorFlow)"},{"level":2,"text":"Description of Code","anchor":"description-of-code","htmlText":"Description of Code"},{"level":2,"text":"Getting Started","anchor":"getting-started","htmlText":"Getting Started"},{"level":2,"text":"How to Train from Scratch","anchor":"how-to-train-from-scratch","htmlText":"How to Train from Scratch"},{"level":2,"text":"How to Train from Scratch in a Distributed Setting","anchor":"how-to-train-from-scratch-in-a-distributed-setting","htmlText":"How to Train from Scratch in a Distributed Setting"},{"level":2,"text":"How to Evaluate","anchor":"how-to-evaluate","htmlText":"How to Evaluate"},{"level":2,"text":"How to Fine-Tune a Pre-Trained Model on a New Task","anchor":"how-to-fine-tune-a-pre-trained-model-on-a-new-task","htmlText":"How to Fine-Tune a Pre-Trained Model on a New Task"},{"level":3,"text":"Getting Started","anchor":"getting-started-1","htmlText":"Getting Started"},{"level":3,"text":"How to Retrain a Trained Model on the Flowers Data","anchor":"how-to-retrain-a-trained-model-on-the-flowers-data","htmlText":"How to Retrain a Trained Model on the Flowers Data"},{"level":2,"text":"How to Construct a New Dataset for Retraining","anchor":"how-to-construct-a-new-dataset-for-retraining","htmlText":"How to Construct a New Dataset for Retraining"},{"level":2,"text":"Practical Considerations for Training a Model","anchor":"practical-considerations-for-training-a-model","htmlText":"Practical Considerations for Training a Model"},{"level":3,"text":"Finding Good Hyperparameters","anchor":"finding-good-hyperparameters","htmlText":"Finding Good Hyperparameters"},{"level":3,"text":"Adjusting Memory Demands","anchor":"adjusting-memory-demands","htmlText":"Adjusting Memory Demands"},{"level":2,"text":"Troubleshooting","anchor":"troubleshooting","htmlText":"Troubleshooting"},{"level":4,"text":"The model runs out of CPU memory.","anchor":"the-model-runs-out-of-cpu-memory","htmlText":"The model runs out of CPU memory."},{"level":4,"text":"The model runs out of GPU memory.","anchor":"the-model-runs-out-of-gpu-memory","htmlText":"The model runs out of GPU memory."},{"level":4,"text":"The model training results in NaN's.","anchor":"the-model-training-results-in-nans","htmlText":"The model training results in NaN's."},{"level":4,"text":"I wish to train a model with a different image size.","anchor":"i-wish-to-train-a-model-with-a-different-image-size","htmlText":"I wish to train a model with a different image size."},{"level":4,"text":"What hardware specification are these hyper-parameters targeted for?","anchor":"what-hardware-specification-are-these-hyper-parameters-targeted-for","htmlText":"What hardware specification are these hyper-parameters targeted for?"},{"level":4,"text":"How do I continue training from a checkpoint in distributed setting?","anchor":"how-do-i-continue-training-from-a-checkpoint-in-distributed-setting","htmlText":"How do I continue training from a checkpoint in distributed setting?"}],"siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad"}},{"displayName":"LICENSE","repoName":"terngrad","refName":"master","path":"LICENSE","preferredFileType":"license","tabName":"Apache-2.0","richText":null,"loaded":false,"timedOut":false,"errorMessage":null,"headerInfo":{"toc":null,"siteNavLoginPath":"/login?return_to=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad"}}],"overviewFilesProcessingTime":0}},"appPayload":{"helpUrl":"https://docs.github.com","findFileWorkerPath":"/assets-cdn/worker/find-file-worker-7d7eb7c71814.js","findInFileWorkerPath":"/assets-cdn/worker/find-in-file-worker-96e76d5fdb2c.js","githubDevUrl":null,"enabled_features":{"copilot_workspace":null,"code_nav_ui_events":false,"overview_shared_code_dropdown_button":false,"react_blob_overlay":false,"accessible_code_button":true,"github_models_repo_integration":false}}}}</script> <div data-target="react-partial.reactRoot"><style data-styled="true" data-styled-version="5.3.11">.iVEunk{margin-top:16px;margin-bottom:16px;}/*!sc*/ .jzuOtQ{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;}/*!sc*/ .bGojzy{margin-bottom:0;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;row-gap:16px;}/*!sc*/ .iNSVHo{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;padding-bottom:16px;padding-top:8px;}/*!sc*/ .bVgnfw{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;gap:8px;}/*!sc*/ @media screen and (max-width:320px){.bVgnfw{-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;}}/*!sc*/ .CEgMp{position:relative;}/*!sc*/ @media screen and (max-width:380px){.CEgMp .ref-selector-button-text-container{max-width:80px;}}/*!sc*/ @media screen and (max-width:320px){.CEgMp{-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;}.CEgMp .overview-ref-selector{width:100%;}.CEgMp .overview-ref-selector > span{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-pack:start;-webkit-justify-content:flex-start;-ms-flex-pack:start;justify-content:flex-start;}.CEgMp .overview-ref-selector > span > span[data-component="text"]{-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;}}/*!sc*/ .gMOVLe[data-size="medium"]{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;min-width:0;}/*!sc*/ .gMOVLe[data-size="medium"] svg{color:var(--fgColor-muted,var(--color-fg-muted,#656d76));}/*!sc*/ .gMOVLe[data-size="medium"] > span{width:inherit;}/*!sc*/ .gUkoLg{-webkit-box-pack:center;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center;}/*!sc*/ .bZBlpz{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;width:100%;}/*!sc*/ .lhTYNA{margin-right:4px;color:var(--fgColor-muted,var(--color-fg-muted,#656d76));}/*!sc*/ .ffLUq{font-size:14px;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;}/*!sc*/ .bmcJak{min-width:0;}/*!sc*/ .fLXEGX{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;}/*!sc*/ @media screen and (max-width:1079px){.fLXEGX{display:none;}}/*!sc*/ .lmSMZJ[data-size="medium"]{color:var(--fgColor-muted,var(--color-fg-muted,#656d76));padding-left:4px;padding-right:4px;}/*!sc*/ .lmSMZJ[data-size="medium"] span[data-component="leadingVisual"]{margin-right:4px !important;}/*!sc*/ .dqfxud{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;}/*!sc*/ @media screen and (min-width:1080px){.dqfxud{display:none;}}/*!sc*/ @media screen and (max-width:543px){.dqfxud{display:none;}}/*!sc*/ .fGwBZA[data-size="medium"][data-no-visuals]{color:var(--fgColor-muted,var(--color-fg-muted,#656d76));}/*!sc*/ .jxTzTd{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;padding-left:8px;gap:8px;}/*!sc*/ .gqqBXN{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;gap:8px;}/*!sc*/ @media screen and (max-width:543px){.gqqBXN{display:none;}}/*!sc*/ .dzXgxt{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;}/*!sc*/ @media screen and (max-width:1011px){.dzXgxt{display:none;}}/*!sc*/ .iWFGlI{margin-left:8px;margin-right:8px;margin:0;}/*!sc*/ .vcvyP{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;min-width:160px;}/*!sc*/ .YUPas{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;}/*!sc*/ @media screen and (min-width:1012px){.YUPas{display:none;}}/*!sc*/ .izFOf{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;}/*!sc*/ @media screen and (min-width:544px){.izFOf{display:none;}}/*!sc*/ .vIPPs{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;gap:16px;}/*!sc*/ .fdROMU{width:100%;border-collapse:separate;border-spacing:0;border:1px solid;border-color:var(--borderColor-default,var(--color-border-default,#d0d7de));border-radius:6px;table-layout:fixed;overflow:unset;}/*!sc*/ .jGKpsv{height:0px;line-height:0px;}/*!sc*/ .jGKpsv tr{height:0px;font-size:0px;}/*!sc*/ .jdgHnn{padding:16px;color:var(--fgColor-muted,var(--color-fg-muted,#656d76));font-size:12px;text-align:left;height:40px;}/*!sc*/ .jdgHnn th{padding-left:16px;background-color:var(--bgColor-muted,var(--color-canvas-subtle,#f6f8fa));}/*!sc*/ .bQivRW{width:100%;border-top-left-radius:6px;}/*!sc*/ @media screen and (min-width:544px){.bQivRW{display:none;}}/*!sc*/ .ldkMIO{width:40%;border-top-left-radius:6px;}/*!sc*/ @media screen and (max-width:543px){.ldkMIO{display:none;}}/*!sc*/ .jMbWeI{text-align:right;padding-right:16px;width:136px;border-top-right-radius:6px;}/*!sc*/ .gpqjiB{color:var(--fgColor-muted,var(--color-fg-muted,#656d76));font-size:12px;height:40px;}/*!sc*/ .dzCJzi{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;-webkit-flex-wrap:wrap;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-pack:justify;-webkit-justify-content:space-between;-ms-flex-pack:justify;justify-content:space-between;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;gap:8px;min-width:273px;padding:8px;}/*!sc*/ @media screen and (min-width:544px){.dzCJzi{-webkit-flex-wrap:nowrap;-ms-flex-wrap:nowrap;flex-wrap:nowrap;}}/*!sc*/ .eNCcrz{text-align:center;vertical-align:center;height:40px;border-top:1px solid;border-color:var(--borderColor-default,var(--color-border-default,#d0d7de));}/*!sc*/ .bHTcCe{border-top:1px solid var(--borderColor-default,var(--color-border-default));cursor:pointer;}/*!sc*/ .csrIcr{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;gap:16px;}/*!sc*/ .bUQNHB{border:1px solid;border-color:var(--borderColor-default,var(--color-border-default,#d0d7de));border-radius:6px;display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;}/*!sc*/ @media screen and (max-width:543px){.bUQNHB{margin-left:-16px;margin-right:-16px;max-width:calc(100% + 32px);}}/*!sc*/ @media screen and (min-width:544px){.bUQNHB{max-width:100%;}}/*!sc*/ .jPdcfu{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;border-bottom:1px solid;border-bottom-color:var(--borderColor-default,var(--color-border-default,#d0d7de));-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding-right:8px;position:-webkit-sticky;position:sticky;top:0;background-color:var(--bgColor-default,var(--color-canvas-default,#ffffff));z-index:1;border-top-left-radius:6px;border-top-right-radius:6px;}/*!sc*/ .iphEWz{-webkit-box-flex:1;-webkit-flex-grow:1;-ms-flex-positive:1;flex-grow:1;border-bottom:none;max-width:100%;padding-left:8px;padding-right:8px;}/*!sc*/ .hUCRAk{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;-webkit-align-items:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;}/*!sc*/ .cwoBXV[data-size="medium"]{color:var(--fgColor-muted,var(--color-fg-subtle,#6e7781));padding-left:8px;padding-right:8px;}/*!sc*/ .QkQOb{padding:32px;overflow:auto;}/*!sc*/ data-styled.g1[id="Box-sc-g0xbh4-0"]{content:"iVEunk,jzuOtQ,bGojzy,iNSVHo,bVgnfw,CEgMp,gMOVLe,gUkoLg,bZBlpz,lhTYNA,ffLUq,bmcJak,fLXEGX,lmSMZJ,dqfxud,fGwBZA,jxTzTd,gqqBXN,dzXgxt,iWFGlI,vcvyP,YUPas,izFOf,vIPPs,fdROMU,jGKpsv,jdgHnn,bQivRW,ldkMIO,jMbWeI,gpqjiB,dzCJzi,eNCcrz,bHTcCe,csrIcr,bUQNHB,jPdcfu,iphEWz,hUCRAk,cwoBXV,QkQOb,"}/*!sc*/ .brGdpi{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;-webkit-clip:rect(0,0,0,0);clip:rect(0,0,0,0);white-space:nowrap;border-width:0;}/*!sc*/ data-styled.g6[id="_VisuallyHidden__VisuallyHidden-sc-11jhm7a-0"]{content:"brGdpi,"}/*!sc*/ .hWlpPn{position:relative;display:inline-block;}/*!sc*/ .hWlpPn::after{position:absolute;z-index:1000000;display:none;padding:0.5em 0.75em;font:normal normal 11px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";-webkit-font-smoothing:subpixel-antialiased;color:var(--tooltip-fgColor,var(--fgColor-onEmphasis,var(--color-fg-on-emphasis,#ffffff)));text-align:center;-webkit-text-decoration:none;text-decoration:none;text-shadow:none;text-transform:none;-webkit-letter-spacing:normal;-moz-letter-spacing:normal;-ms-letter-spacing:normal;letter-spacing:normal;word-wrap:break-word;white-space:pre;pointer-events:none;content:attr(aria-label);background:var(--tooltip-bgColor,var(--bgColor-emphasis,var(--color-neutral-emphasis-plus,#24292f)));border-radius:6px;opacity:0;}/*!sc*/ @-webkit-keyframes tooltip-appear{from{opacity:0;}to{opacity:1;}}/*!sc*/ @keyframes tooltip-appear{from{opacity:0;}to{opacity:1;}}/*!sc*/ .hWlpPn:hover::after,.hWlpPn:active::after,.hWlpPn:focus::after,.hWlpPn:focus-within::after{display:inline-block;-webkit-text-decoration:none;text-decoration:none;-webkit-animation-name:tooltip-appear;animation-name:tooltip-appear;-webkit-animation-duration:0.1s;animation-duration:0.1s;-webkit-animation-fill-mode:forwards;animation-fill-mode:forwards;-webkit-animation-timing-function:ease-in;animation-timing-function:ease-in;-webkit-animation-delay:0s;animation-delay:0s;}/*!sc*/ .hWlpPn.tooltipped-no-delay:hover::after,.hWlpPn.tooltipped-no-delay:active::after,.hWlpPn.tooltipped-no-delay:focus::after,.hWlpPn.tooltipped-no-delay:focus-within::after{-webkit-animation-delay:0s;animation-delay:0s;}/*!sc*/ .hWlpPn.tooltipped-multiline:hover::after,.hWlpPn.tooltipped-multiline:active::after,.hWlpPn.tooltipped-multiline:focus::after,.hWlpPn.tooltipped-multiline:focus-within::after{display:table-cell;}/*!sc*/ .hWlpPn.tooltipped-s::after,.hWlpPn.tooltipped-se::after,.hWlpPn.tooltipped-sw::after{top:100%;right:50%;margin-top:6px;}/*!sc*/ .hWlpPn.tooltipped-se::after{right:auto;left:50%;margin-left:-16px;}/*!sc*/ .hWlpPn.tooltipped-sw::after{margin-right:-16px;}/*!sc*/ .hWlpPn.tooltipped-n::after,.hWlpPn.tooltipped-ne::after,.hWlpPn.tooltipped-nw::after{right:50%;bottom:100%;margin-bottom:6px;}/*!sc*/ .hWlpPn.tooltipped-ne::after{right:auto;left:50%;margin-left:-16px;}/*!sc*/ .hWlpPn.tooltipped-nw::after{margin-right:-16px;}/*!sc*/ .hWlpPn.tooltipped-s::after,.hWlpPn.tooltipped-n::after{-webkit-transform:translateX(50%);-ms-transform:translateX(50%);transform:translateX(50%);}/*!sc*/ .hWlpPn.tooltipped-w::after{right:100%;bottom:50%;margin-right:6px;-webkit-transform:translateY(50%);-ms-transform:translateY(50%);transform:translateY(50%);}/*!sc*/ .hWlpPn.tooltipped-e::after{bottom:50%;left:100%;margin-left:6px;-webkit-transform:translateY(50%);-ms-transform:translateY(50%);transform:translateY(50%);}/*!sc*/ .hWlpPn.tooltipped-multiline::after{width:-webkit-max-content;width:-moz-max-content;width:max-content;max-width:250px;word-wrap:break-word;white-space:pre-line;border-collapse:separate;}/*!sc*/ .hWlpPn.tooltipped-multiline.tooltipped-s::after,.hWlpPn.tooltipped-multiline.tooltipped-n::after{right:auto;left:50%;-webkit-transform:translateX(-50%);-ms-transform:translateX(-50%);transform:translateX(-50%);}/*!sc*/ .hWlpPn.tooltipped-multiline.tooltipped-w::after,.hWlpPn.tooltipped-multiline.tooltipped-e::after{right:100%;}/*!sc*/ .hWlpPn.tooltipped-align-right-2::after{right:0;margin-right:0;}/*!sc*/ .hWlpPn.tooltipped-align-left-2::after{left:0;margin-left:0;}/*!sc*/ data-styled.g17[id="Tooltip__TooltipBase-sc-17tf59c-0"]{content:"hWlpPn,"}/*!sc*/ .liVpTx{display:inline-block;overflow:hidden;text-overflow:ellipsis;vertical-align:top;white-space:nowrap;max-width:125px;}/*!sc*/ data-styled.g19[id="Truncate__StyledTruncate-sc-23o1d2-0"]{content:"liVpTx,"}/*!sc*/ </style> <!-- --> <!-- --> <div class="Box-sc-g0xbh4-0 iVEunk"><div class="Box-sc-g0xbh4-0 jzuOtQ"><div class="Box-sc-g0xbh4-0 bGojzy"></div></div><div class="Box-sc-g0xbh4-0 iNSVHo"><div class="Box-sc-g0xbh4-0 bVgnfw"><div class="Box-sc-g0xbh4-0 CEgMp"><button type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" aria-label="master branch" data-testid="anchor-button" class="Box-sc-g0xbh4-0 gMOVLe prc-Button-ButtonBase-c50BI overview-ref-selector width-full" data-loading="false" data-size="medium" data-variant="default" aria-describedby="branch-picker-repos-header-ref-selector-loading-announcement" id="branch-picker-repos-header-ref-selector"><span data-component="buttonContent" class="Box-sc-g0xbh4-0 gUkoLg prc-Button-ButtonContent-HKbr-"><span data-component="text" class="prc-Button-Label-pTQ3x"><div class="Box-sc-g0xbh4-0 bZBlpz"><div class="Box-sc-g0xbh4-0 lhTYNA"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></div><div class="Box-sc-g0xbh4-0 ffLUq ref-selector-button-text-container"><span class="Box-sc-g0xbh4-0 bmcJak prc-Text-Text-0ima0"> <!-- -->master</span></div></div></span><span data-component="trailingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><button hidden="" data-hotkey-scope="read-only-cursor-text-area"></button></div><div class="Box-sc-g0xbh4-0 fLXEGX"><a style="--button-color:fg.muted" type="button" href="/wenwei202/terngrad/branches" class="Box-sc-g0xbh4-0 lmSMZJ prc-Button-ButtonBase-c50BI" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rclab:-loading-announcement"><span data-component="buttonContent" class="Box-sc-g0xbh4-0 gUkoLg prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Branches</span></span></a><a style="--button-color:fg.muted" type="button" href="/wenwei202/terngrad/tags" class="Box-sc-g0xbh4-0 lmSMZJ prc-Button-ButtonBase-c50BI" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rklab:-loading-announcement"><span data-component="buttonContent" class="Box-sc-g0xbh4-0 gUkoLg prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-tag" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Tags</span></span></a></div><div class="Box-sc-g0xbh4-0 dqfxud"><a style="--button-color:fg.muted" type="button" aria-label="Go to Branches page" href="/wenwei202/terngrad/branches" class="Box-sc-g0xbh4-0 fGwBZA prc-Button-ButtonBase-c50BI" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-describedby=":Relab:-loading-announcement"><svg aria-hidden="true" focusable="false" class="octicon octicon-git-branch" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M9.5 3.25a2.25 2.25 0 1 1 3 2.122V6A2.5 2.5 0 0 1 10 8.5H6a1 1 0 0 0-1 1v1.128a2.251 2.251 0 1 1-1.5 0V5.372a2.25 2.25 0 1 1 1.5 0v1.836A2.493 2.493 0 0 1 6 7h4a1 1 0 0 0 1-1v-.628A2.25 2.25 0 0 1 9.5 3.25Zm-6 0a.75.75 0 1 0 1.5 0 .75.75 0 0 0-1.5 0Zm8.25-.75a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5ZM4.25 12a.75.75 0 1 0 0 1.5.75.75 0 0 0 0-1.5Z"></path></svg></a><a style="--button-color:fg.muted" type="button" aria-label="Go to Tags page" href="/wenwei202/terngrad/tags" class="Box-sc-g0xbh4-0 fGwBZA prc-Button-ButtonBase-c50BI" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="invisible" aria-describedby=":Rmlab:-loading-announcement"><svg aria-hidden="true" focusable="false" class="octicon octicon-tag" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1 7.775V2.75C1 1.784 1.784 1 2.75 1h5.025c.464 0 .91.184 1.238.513l6.25 6.25a1.75 1.75 0 0 1 0 2.474l-5.026 5.026a1.75 1.75 0 0 1-2.474 0l-6.25-6.25A1.752 1.752 0 0 1 1 7.775Zm1.5 0c0 .066.026.13.073.177l6.25 6.25a.25.25 0 0 0 .354 0l5.025-5.025a.25.25 0 0 0 0-.354l-6.25-6.25a.25.25 0 0 0-.177-.073H2.75a.25.25 0 0 0-.25.25ZM6 5a1 1 0 1 1 0 2 1 1 0 0 1 0-2Z"></path></svg></a></div></div><div class="Box-sc-g0xbh4-0 jxTzTd"><div class="Box-sc-g0xbh4-0 gqqBXN"><div class="Box-sc-g0xbh4-0 dzXgxt"><!--$--><div class="Box-sc-g0xbh4-0 iWFGlI"><span class="Box-sc-g0xbh4-0 vcvyP TextInput-wrapper prc-components-TextInputWrapper-i1ofR prc-components-TextInputBaseWrapper-ueK9q" data-leading-visual="true" data-trailing-visual="true" aria-busy="false"><span class="TextInput-icon" id=":R2j5ab:" aria-hidden="true"><svg aria-hidden="true" focusable="false" class="octicon octicon-search" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M10.68 11.74a6 6 0 0 1-7.922-8.982 6 6 0 0 1 8.982 7.922l3.04 3.04a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215ZM11.5 7a4.499 4.499 0 1 0-8.997 0A4.499 4.499 0 0 0 11.5 7Z"></path></svg></span><input type="text" aria-label="Go to file" role="combobox" aria-controls="file-results-list" aria-expanded="false" aria-haspopup="dialog" autoCorrect="off" spellcheck="false" placeholder="Go to file" aria-describedby=":R2j5ab: :R2j5abH1:" data-component="input" class="prc-components-Input-Ic-y8" value=""/><span class="TextInput-icon" id=":R2j5abH1:" aria-hidden="true"></span></span></div><!--/$--></div><div class="Box-sc-g0xbh4-0 YUPas"><button type="button" class="prc-Button-ButtonBase-c50BI" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-describedby=":Rr5ab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="text" class="prc-Button-Label-pTQ3x">Go to file</span></span></button></div><div class="react-directory-add-file-icon"></div><div class="react-directory-remove-file-icon"></div></div><button type="button" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-c50BI" data-loading="false" data-size="medium" data-variant="primary" aria-describedby=":R55ab:-loading-announcement" id=":R55ab:"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-code hide-sm" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.749.749 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.749.749 0 0 1 .734.215Zm-6.56 0a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Code</span><span data-component="trailingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-triangle-down" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m4.427 7.427 3.396 3.396a.25.25 0 0 0 .354 0l3.396-3.396A.25.25 0 0 0 11.396 7H4.604a.25.25 0 0 0-.177.427Z"></path></svg></span></span></button><div class="Box-sc-g0xbh4-0 izFOf"><button data-component="IconButton" type="button" aria-label="Open more actions menu" aria-haspopup="true" aria-expanded="false" tabindex="0" class="prc-Button-ButtonBase-c50BI prc-Button-IconButton-szpyj" data-loading="false" data-no-visuals="true" data-size="medium" data-variant="default" aria-describedby=":R75ab:-loading-announcement" id=":R75ab:"><svg aria-hidden="true" focusable="false" class="octicon octicon-kebab-horizontal" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3ZM1.5 9a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Zm13 0a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"></path></svg></button></div></div></div><div class="Box-sc-g0xbh4-0 vIPPs"><div data-hpc="true"><button hidden="" data-testid="focus-next-element-button" data-hotkey="j"></button><button hidden="" data-testid="focus-previous-element-button" data-hotkey="k"></button><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading" id="folders-and-files">Folders and files</h2><table aria-labelledby="folders-and-files" class="Box-sc-g0xbh4-0 fdROMU"><thead class="Box-sc-g0xbh4-0 jGKpsv"><tr class="Box-sc-g0xbh4-0 jdgHnn"><th colSpan="2" class="Box-sc-g0xbh4-0 bQivRW"><span class="text-bold">Name</span></th><th colSpan="1" class="Box-sc-g0xbh4-0 ldkMIO"><span class="text-bold">Name</span></th><th class="hide-sm"><div title="Last commit message" class="Truncate__StyledTruncate-sc-23o1d2-0 liVpTx width-fit"><span class="text-bold">Last commit message</span></div></th><th colSpan="1" class="Box-sc-g0xbh4-0 jMbWeI"><div title="Last commit date" class="Truncate__StyledTruncate-sc-23o1d2-0 liVpTx width-fit"><span class="text-bold">Last commit date</span></div></th></tr></thead><tbody><tr class="Box-sc-g0xbh4-0 gpqjiB"><td colSpan="3" class="bgColor-muted p-1 rounded-top-2"><div class="Box-sc-g0xbh4-0 dzCJzi"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading">Latest commit</h2><div style="width:120px" class="Skeleton Skeleton--text" data-testid="loading"> </div><div class="d-flex flex-shrink-0 gap-2"><div data-testid="latest-commit-details" class="d-none d-sm-flex flex-items-center"></div><div class="d-flex gap-2"><h2 class="sr-only ScreenReaderHeading-module__userSelectNone--vW4Cq prc-Heading-Heading-6CmGO" data-testid="screen-reader-heading">History</h2><a href="/wenwei202/terngrad/commits/master/" class="prc-Button-ButtonBase-c50BI d-none d-lg-flex LinkButton-module__code-view-link-button--xvCGA flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby=":Raqj8pab:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x"><span class="fgColor-default">280 Commits</span></span></span></a><div class="d-sm-none"></div><div class="d-flex d-lg-none"><span role="tooltip" aria-label="280 Commits" id="history-icon-button-tooltip" class="Tooltip__TooltipBase-sc-17tf59c-0 hWlpPn tooltipped-n"><a href="/wenwei202/terngrad/commits/master/" class="prc-Button-ButtonBase-c50BI LinkButton-module__code-view-link-button--xvCGA flex-items-center fgColor-default" data-loading="false" data-size="small" data-variant="invisible" aria-describedby=":R1iqj8pab:-loading-announcement history-icon-button-tooltip"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-history" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="m.427 1.927 1.215 1.215a8.002 8.002 0 1 1-1.6 5.685.75.75 0 1 1 1.493-.154 6.5 6.5 0 1 0 1.18-4.458l1.358 1.358A.25.25 0 0 1 3.896 6H.25A.25.25 0 0 1 0 5.75V2.104a.25.25 0 0 1 .427-.177ZM7.75 4a.75.75 0 0 1 .75.75v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5A.75.75 0 0 1 7.75 4Z"></path></svg></span></span></a></span></div></div></div></div></td></tr><tr class="react-directory-row undefined" id="folder-row-0"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="slim" aria-label="slim, (Directory)" class="Link--primary" href="/wenwei202/terngrad/tree/master/slim">slim</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="slim" aria-label="slim, (Directory)" class="Link--primary" href="/wenwei202/terngrad/tree/master/slim">slim</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-1"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="terngrad" aria-label="terngrad, (Directory)" class="Link--primary" href="/wenwei202/terngrad/tree/master/terngrad">terngrad</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file-directory-fill icon-directory" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M1.75 1A1.75 1.75 0 0 0 0 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0 0 16 13.25v-8.5A1.75 1.75 0 0 0 14.25 3H7.5a.25.25 0 0 1-.2-.1l-.9-1.2C6.07 1.26 5.55 1 5 1H1.75Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="terngrad" aria-label="terngrad, (Directory)" class="Link--primary" href="/wenwei202/terngrad/tree/master/terngrad">terngrad</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-2"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title=".gitignore" aria-label=".gitignore, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/.gitignore">.gitignore</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title=".gitignore" aria-label=".gitignore, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/.gitignore">.gitignore</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-3"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="LICENSE" aria-label="LICENSE, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/LICENSE">LICENSE</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="LICENSE" aria-label="LICENSE, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/LICENSE">LICENSE</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-4"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="NIPS17-TernGrad-slides-v3.pdf" aria-label="NIPS17-TernGrad-slides-v3.pdf, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/NIPS17-TernGrad-slides-v3.pdf">NIPS17-TernGrad-slides-v3.pdf</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="NIPS17-TernGrad-slides-v3.pdf" aria-label="NIPS17-TernGrad-slides-v3.pdf, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/NIPS17-TernGrad-slides-v3.pdf">NIPS17-TernGrad-slides-v3.pdf</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-5"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="Poster_Wen_NIPS2017.pdf" aria-label="Poster_Wen_NIPS2017.pdf, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/Poster_Wen_NIPS2017.pdf">Poster_Wen_NIPS2017.pdf</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="Poster_Wen_NIPS2017.pdf" aria-label="Poster_Wen_NIPS2017.pdf, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/Poster_Wen_NIPS2017.pdf">Poster_Wen_NIPS2017.pdf</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="react-directory-row undefined" id="folder-row-6"><td class="react-directory-row-name-cell-small-screen" colSpan="2"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="README.md" aria-label="README.md, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/README.md">README.md</a></div></div></div></div></td><td class="react-directory-row-name-cell-large-screen" colSpan="1"><div class="react-directory-filename-column"><svg aria-hidden="true" focusable="false" class="octicon octicon-file color-fg-muted" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M2 1.75C2 .784 2.784 0 3.75 0h6.586c.464 0 .909.184 1.237.513l2.914 2.914c.329.328.513.773.513 1.237v9.586A1.75 1.75 0 0 1 13.25 16h-9.5A1.75 1.75 0 0 1 2 14.25Zm1.75-.25a.25.25 0 0 0-.25.25v12.5c0 .138.112.25.25.25h9.5a.25.25 0 0 0 .25-.25V6h-2.75A1.75 1.75 0 0 1 9 4.25V1.5Zm6.75.062V4.25c0 .138.112.25.25.25h2.688l-.011-.013-2.914-2.914-.013-.011Z"></path></svg><div class="overflow-hidden"><div class="react-directory-filename-cell"><div class="react-directory-truncate"><a title="README.md" aria-label="README.md, (File)" class="Link--primary" href="/wenwei202/terngrad/blob/master/README.md">README.md</a></div></div></div></div></td><td class="react-directory-row-commit-cell"><div class="Skeleton Skeleton--text"> </div></td><td><div class="Skeleton Skeleton--text"> </div></td></tr><tr class="Box-sc-g0xbh4-0 eNCcrz d-none" data-testid="view-all-files-row"><td colSpan="3" class="Box-sc-g0xbh4-0 bHTcCe"><div><button class="prc-Link-Link-85e08">View all files</button></div></td></tr></tbody></table></div><div class="Box-sc-g0xbh4-0 csrIcr"><div class="Box-sc-g0xbh4-0 bUQNHB"><div itemscope="" itemType="https://schema.org/abstract" class="Box-sc-g0xbh4-0 jPdcfu"><h2 class="_VisuallyHidden__VisuallyHidden-sc-11jhm7a-0 brGdpi">Repository files navigation</h2><nav class="Box-sc-g0xbh4-0 iphEWz prc-components-UnderlineWrapper-oOh5J" aria-label="Repository files"><ul class="prc-components-UnderlineItemList-b23Hf" role="list"><li class="Box-sc-g0xbh4-0 hUCRAk"><a class="prc-components-UnderlineItem-lJsg-" href="#" aria-current="page"><span data-component="icon"><svg aria-hidden="true" focusable="false" class="octicon octicon-book" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path></svg></span><span data-component="text" data-content="README">README</span></a></li><li class="Box-sc-g0xbh4-0 hUCRAk"><a class="prc-components-UnderlineItem-lJsg-" href="#"><span data-component="icon"><svg aria-hidden="true" focusable="false" class="octicon octicon-law" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M8.75.75V2h.985c.304 0 .603.08.867.231l1.29.736c.038.022.08.033.124.033h2.234a.75.75 0 0 1 0 1.5h-.427l2.111 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.006.005-.01.01-.045.04c-.21.176-.441.327-.686.45C14.556 10.78 13.88 11 13 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L12.178 4.5h-.162c-.305 0-.604-.079-.868-.231l-1.29-.736a.245.245 0 0 0-.124-.033H8.75V13h2.5a.75.75 0 0 1 0 1.5h-6.5a.75.75 0 0 1 0-1.5h2.5V3.5h-.984a.245.245 0 0 0-.124.033l-1.289.737c-.265.15-.564.23-.869.23h-.162l2.112 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.016.015-.045.04c-.21.176-.441.327-.686.45C4.556 10.78 3.88 11 3 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L2.178 4.5H1.75a.75.75 0 0 1 0-1.5h2.234a.249.249 0 0 0 .125-.033l1.288-.737c.265-.15.564-.23.869-.23h.984V.75a.75.75 0 0 1 1.5 0Zm2.945 8.477c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L13 6.327Zm-10 0c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L3 6.327Z"></path></svg></span><span data-component="text" data-content="Apache-2.0 license">Apache-2.0 license</span></a></li></ul></nav><button style="--button-color:fg.subtle" type="button" aria-label="Outline" aria-haspopup="true" aria-expanded="false" tabindex="0" class="Box-sc-g0xbh4-0 cwoBXV prc-Button-ButtonBase-c50BI" data-loading="false" data-size="medium" data-variant="invisible" aria-describedby=":Rr9ab:-loading-announcement" id=":Rr9ab:"><svg aria-hidden="true" focusable="false" class="octicon octicon-list-unordered" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align:text-bottom"><path d="M5.75 2.5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5Zm0 5h8.5a.75.75 0 0 1 0 1.5h-8.5a.75.75 0 0 1 0-1.5ZM2 14a1 1 0 1 1 0-2 1 1 0 0 1 0 2Zm1-6a1 1 0 1 1-2 0 1 1 0 0 1 2 0ZM2 4a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg></button></div><div class="Box-sc-g0xbh4-0 QkQOb js-snippet-clipboard-copy-unpositioned undefined" data-hpc="true"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Introduction</h1><a id="user-content-introduction" class="anchor" aria-label="Permalink: Introduction" href="#introduction"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">This repo is the TensorFlow code for our oral paper in NIPS 2017 (<a href="https://papers.nips.cc/paper/6749-terngrad-ternary-gradients-to-reduce-communication-in-distributed-deep-learning.pdf" rel="nofollow">TernGrad: Ternary Gradients to Reduce Communication in Distributed Deep Learning</a>).</p> <p dir="auto"><strong>The implementations of quantization and de-quantization operators are now available in <a href="https://github.com/pytorch/pytorch/blob/master/caffe2/operators/fused_rowwise_random_quantization_ops.cc">Caffe2/Pytorch 1.0</a> (<a href="https://github.com/pytorch/pytorch/blob/master/caffe2/python/operator_test/rand_quantization_op_test.py">an example</a>). TernGrad is landed into Facebook AI platforms in production to overcome communication bottleneck in large-scale training.</strong></p> <p dir="auto"><a href="https://www.youtube.com/watch?v=WWWQXTb_69c&feature=youtu.be&t=20s" rel="nofollow">video</a>, <a href="/wenwei202/terngrad/blob/master/Poster_Wen_NIPS2017.pdf">poster</a>, <a href="/wenwei202/terngrad/blob/master/NIPS17-TernGrad-slides-v3.pdf">slides</a></p> <p dir="auto">This is a modified copy of TensorFlow <a href="https://github.com/tensorflow/models/tree/master/inception">inception</a> (with original contributions kept).</p> <p dir="auto"><em>In this workspace, <code>inception</code> refers to all types of neural networks in a general way.</em></p> <p dir="auto"><em>Note that there is name abuse because of history reasons. All "bingrad/binary gradient/binarizing" in code comments, help info and filenames essentially refers to "terngrad/ternary gradient/ternarizing". Will update them, but the code is correct and is exactly for terngrad</em></p> <p dir="auto"><em>The code is for algorithm evaluation only, i.e., -1,0,1 are still represented in floating precision. We are hacking TensorFlow/Caffe2 to support encoder/decoder of ternary gradients.</em></p> <p dir="auto">For the interests in accelerating the inference of DNNs (<a href="http://papers.nips.cc/paper/6504-learning-structured-sparsity-in-deep-neural-networks.pdf" rel="nofollow">NIPS 2016</a>, <a href="https://arxiv.org/abs/1703.09746" rel="nofollow">ICCV 2017</a> and <a href="https://arxiv.org/abs/1709.05027" rel="nofollow">ICLR 2018</a>) using sparsity and low rank approximation, the code links are <a href="https://github.com/wenwei202/caffe">here</a> and <a href="https://github.com/wenwei202/iss-rnns">there</a>.</p> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Dependencies</h1><a id="user-content-dependencies" class="anchor" aria-label="Permalink: Dependencies" href="#dependencies"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Tested stable dependencies:</p> <ul dir="auto"> <li>python 2.7 (Anaconda)</li> <li>Tensorflow v1.0.0 and v1.3.0</li> <li>cudnn 5.1.5</li> <li>bazel release 0.4.4</li> </ul> <p dir="auto">Pending to test by python 3.6.1 (Anaconda) and Tensorflow 1.1.0 (installed from python wheel). Known issues (mainly because of update to python 3):</p> <ul dir="auto"> <li>use <code>pickle</code> instead of <code>cPickle</code> python package</li> <li>use <code>range</code> instead of <code>xrange</code></li> <li>use <code>dict.items</code> instead of <code>dict.iteritems</code></li> <li><code>TypeError: 'RGB' has type str, but expected one of: bytes</code>: use <code>b'RGB'</code> instead of <code>'RGB'</code></li> <li>...</li> </ul> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Build all</h1><a id="user-content-build-all" class="anchor" aria-label="Permalink: Build all" href="#build-all"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/terngrad ./build_all.sh"><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/terngrad ./build_all.sh </code></pre></div> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Dataset preparation</h1><a id="user-content-dataset-preparation" class="anchor" aria-label="Permalink: Dataset preparation" href="#dataset-preparation"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Download and generate mnist TFRecord</h2><a id="user-content-download-and-generate-mnist-tfrecord" class="anchor" aria-label="Permalink: Download and generate mnist TFRecord" href="#download-and-generate-mnist-tfrecord"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/slim # Generate train-mnist.tfrecord and test-mnist.tfrecord export DATA_PATH="${HOME}/dataset/mnist-data/" python download_and_convert_data.py --dataset_name mnist --dataset_dir ${DATA_PATH}"><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/slim # Generate train-mnist.tfrecord and test-mnist.tfrecord export DATA_PATH="${HOME}/dataset/mnist-data/" python download_and_convert_data.py --dataset_name mnist --dataset_dir ${DATA_PATH} </code></pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Download and generate cifar-10 TFRecord</h2><a id="user-content-download-and-generate-cifar-10-tfrecord" class="anchor" aria-label="Permalink: Download and generate cifar-10 TFRecord" href="#download-and-generate-cifar-10-tfrecord"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/slim # Generate train-cifar10.tfrecord and test-cifar10.tfrecord export DATA_PATH="${HOME}/dataset/cifar10-data/" # the directory of database python download_and_convert_data.py --dataset_name cifar10 --dataset_dir ${DATA_PATH} # Instead of putting all training examples in one tfrecord file, we can split them by enabling --shard # This is useful for distributed training by date parallelsim, where we should split data across nodes # Generate train-xxxxx-of-xxxxx (1000 shards in default) and test-00000-of-00001 tfrecord shards export DATA_PATH="${HOME}/dataset/cifar10-shard-data/" # the directory of database python download_and_convert_data.py \ --dataset_name cifar10 \ --dataset_dir ${DATA_PATH} \ --shard True"><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/slim # Generate train-cifar10.tfrecord and test-cifar10.tfrecord export DATA_PATH="${HOME}/dataset/cifar10-data/" # the directory of database python download_and_convert_data.py --dataset_name cifar10 --dataset_dir ${DATA_PATH} # Instead of putting all training examples in one tfrecord file, we can split them by enabling --shard # This is useful for distributed training by date parallelsim, where we should split data across nodes # Generate train-xxxxx-of-xxxxx (1000 shards in default) and test-00000-of-00001 tfrecord shards export DATA_PATH="${HOME}/dataset/cifar10-shard-data/" # the directory of database python download_and_convert_data.py \ --dataset_name cifar10 \ --dataset_dir ${DATA_PATH} \ --shard True </code></pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Download and generate ImageNet TFRecord</h2><a id="user-content-download-and-generate-imagenet-tfrecord" class="anchor" aria-label="Permalink: Download and generate ImageNet TFRecord" href="#download-and-generate-imagenet-tfrecord"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Before generating, <code>RAW_PIXEL=True</code> in <code>${TERNGRAD_ROOT}/terngrad/inception/data/download_and_preprocess_imagenet.sh</code> can enable storing raw RGB pixels of images into TFRecord.</p> <p dir="auto">Storing raw pixels can save JPG decoding time but burden storage read bandwidth. Set <code>RAW_PIXEL=True</code> if high-speed external storage (like SSD) is used but decoder like in CPU cannot feed as fast as training (like in multi-GPUs).</p> <p dir="auto">When <code>RAW_PIXEL=True</code>, setting <code>RESIZE_DIMEN</code> to a positive value enables image resizing before writing them to TFRecord files.</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="# location of where to place the ImageNet data # If ILSVRC2012_img_train.tar and ILSVRC2012_img_val.tar were downloaded before, # copy them in ${DATA_DIR} to save time DATA_DIR=/tmp/ # build the preprocessing script. bazel build inception/download_and_preprocess_imagenet # run it bazel-bin/inception/download_and_preprocess_imagenet "${DATA_DIR}""><pre class="notranslate"><code># location of where to place the ImageNet data # If ILSVRC2012_img_train.tar and ILSVRC2012_img_val.tar were downloaded before, # copy them in ${DATA_DIR} to save time DATA_DIR=/tmp/ # build the preprocessing script. bazel build inception/download_and_preprocess_imagenet # run it bazel-bin/inception/download_and_preprocess_imagenet "${DATA_DIR}" </code></pre></div> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Examples on multi-gpu mode</h1><a id="user-content-examples-on-multi-gpu-mode" class="anchor" aria-label="Permalink: Examples on multi-gpu mode" href="#examples-on-multi-gpu-mode"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Training CifarNet by TernGrad with Adam</h2><a id="user-content-training-cifarnet-by-terngrad-with-adam" class="anchor" aria-label="Permalink: Training CifarNet by TernGrad with Adam" href="#training-cifarnet-by-terngrad-with-adam"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/terngrad ./run_multi_gpus_cifar10.sh"><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/terngrad ./run_multi_gpus_cifar10.sh </code></pre></div> <p dir="auto"><a href="/wenwei202/terngrad/blob/master/terngrad/run_multi_gpus_cifar10.sh#L5-L33">run_multi_gpus_cifar10.sh</a> is a training script on cifar-10, which</p> <ul dir="auto"> <li>creates a subfolder under <code>$ROOT_WORKSPACE/${DATASET_NAME}_xxx/</code> to store the training data (<code>${ROOT_WORKSPACE}/${DATASET_NAME}_training_data/</code>), evaluating data (<code>${ROOT_WORKSPACE}/${DATASET_NAME}_eval_data/</code>) and logs (<code>${ROOT_WORKSPACE}/${DATASET_NAME}_info/</code>). The subfolder name or log filename is similar to <code>cifar10_cifar10_alexnet_24_adam_1_0.0002_2.5_0_0.004_0.9_1_128_2_Tue_Sep_19_15-27-51_EDT_2017</code>.</li> <li>starts training and</li> <li>starts evaluating.<br> You can change those environments to play.</li> </ul> <p dir="auto">Use <code>--help</code> to check descriptions for usage of python executables. For example,</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="bazel-bin/inception/cifar10_train --help"><pre class="notranslate"><code>bazel-bin/inception/cifar10_train --help </code></pre></div> <p dir="auto">Some important configurations related to TernGrad:</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="--size_to_binarize SIZE_TO_BINARIZE The min number of parameters in a Variable (weights/biases) to enable ternarizing this Variable. 1 means ternarizing all. You can use this to exclude some small Variables --num_gpus NUM_GPUS How many GPUs to use. --num_nodes NUM_NODES How many virtual nodes to use. One GPU can have multiple nodes This enables emulating N workers in M GPUs, where N > M This is a good feature to verify TernGrad algorithm when multiple GPUs are unavailable --grad_bits [32/1] The number of gradient bits. Either 32 or 1. 32 for floating, and 1 for terngrad (I know ternary is not 1 bit. This is just an argument to use either floating or terngrad. We may consider to extend it to more options ranging from terngrad to floating, like 2 bits, 4 bits, 8 bits, etc) --clip_factor CLIP_FACTOR The factor of stddev to clip gradients (0.0 means no clipping). This is the value of c in gradient clipping technique. 2.5 works well in general --quantize_logits [True/False] --noquantize_logits If quantize the gradients in the last logits layer. (sometimes, a skew distribution of gradients in last layer may affect the effectiveness of terngrad. asymmetric ternary levels may be more effective)"><pre class="notranslate"><code>--size_to_binarize SIZE_TO_BINARIZE The min number of parameters in a Variable (weights/biases) to enable ternarizing this Variable. 1 means ternarizing all. You can use this to exclude some small Variables --num_gpus NUM_GPUS How many GPUs to use. --num_nodes NUM_NODES How many virtual nodes to use. One GPU can have multiple nodes This enables emulating N workers in M GPUs, where N > M This is a good feature to verify TernGrad algorithm when multiple GPUs are unavailable --grad_bits [32/1] The number of gradient bits. Either 32 or 1. 32 for floating, and 1 for terngrad (I know ternary is not 1 bit. This is just an argument to use either floating or terngrad. We may consider to extend it to more options ranging from terngrad to floating, like 2 bits, 4 bits, 8 bits, etc) --clip_factor CLIP_FACTOR The factor of stddev to clip gradients (0.0 means no clipping). This is the value of c in gradient clipping technique. 2.5 works well in general --quantize_logits [True/False] --noquantize_logits If quantize the gradients in the last logits layer. (sometimes, a skew distribution of gradients in last layer may affect the effectiveness of terngrad. asymmetric ternary levels may be more effective) </code></pre></div> <p dir="auto">More explanations are also covered in <a href="/wenwei202/terngrad/blob/master/terngrad/run_multi_gpus_cifar10.sh#L5-L33">run_multi_gpus_cifar10.sh</a>.</p> <p dir="auto">More training bash scripts are in <a href="/wenwei202/terngrad/blob/master/terngrad">terngrad</a>, which have similar arguments.</p> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Examples on distributed-node mode</h1><a id="user-content-examples-on-distributed-node-mode" class="anchor" aria-label="Permalink: Examples on distributed-node mode" href="#examples-on-distributed-node-mode"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">ssh setup</h2><a id="user-content-ssh-setup" class="anchor" aria-label="Permalink: ssh setup" href="#ssh-setup"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">We provide a single script to remotely launch all workers and parameter servers. To authorize access to each other, all machines must share the same ssh key. Please follow this <a href="https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/">tutorial</a> to setup. Moreover, make sure you have added the content in the public key (<code>~/.ssh/id_rsa.pub</code>) to <code>~/.ssh/authorized_keys</code>. In some systems, you also need to add the public key to <code>~/.ssh/authorized_keys2</code>. After this, you can simply copy keys (<code>~/.ssh/id_rsa</code> and <code>~/.ssh/id_rsa.pub</code>) and authorizing files (<code>~/.ssh/authorized_keys</code> and <code>~/.ssh/authorized_keys2</code>) to all machines. At the first run, you may need to answer <code>yes</code> to</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="The authenticity of host '10.236.176.29 (10.236.176.29)' can't be established. ECDSA key fingerprint is SHA256:jkfjkdslajfklsjaflkjs/jowufuf98e8e8eu9. Are you sure you want to continue connecting (yes/no)?"><pre class="notranslate"><code>The authenticity of host '10.236.176.29 (10.236.176.29)' can't be established. ECDSA key fingerprint is SHA256:jkfjkdslajfklsjaflkjs/jowufuf98e8e8eu9. Are you sure you want to continue connecting (yes/no)? </code></pre></div> <p dir="auto">Our bash script uses <code>ssh</code> to login and start worker/ps. If some variables are configured in <code>.bashrc</code> and are necessary for training (e.g., the <code>PATH</code> of anaconda), you may need to source <code>~/.bashrc</code> in <code>~/.bash_profile</code> or <code>~/.profile</code> by adding</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="if [ -f ~/.bashrc ]; then . ~/.bashrc fi"><pre class="notranslate"><code>if [ -f ~/.bashrc ]; then . ~/.bashrc fi </code></pre></div> <p dir="auto">In some linux distributions (e.g. ubuntu) you may need to comment</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="# If not running interactively, don't do anything case $- in *i*) ;; *) return;; esac"><pre class="notranslate"><code># If not running interactively, don't do anything case $- in *i*) ;; *) return;; esac </code></pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">A toy example</h2><a id="user-content-a-toy-example" class="anchor" aria-label="Permalink: A toy example" href="#a-toy-example"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto"><a href="/wenwei202/terngrad/blob/master/terngrad/run_dist_cifar10.sh">run_dist_cifar10.sh</a> is a toy example by launching one parameter server and two workers in <code>localhost</code>. Before start, we must split cifar-10 dataset to two parts: <code>$HOME/dataset/cifar10-data-shard-500-999</code> and <code>$HOME/dataset/cifar10-data-shard-0-499</code>, which each worker paralell fetches and trains its model replica.</p> <p dir="auto">The python executable is <code>bazel-bin/inception/cifar10_distributed_train</code>, of which most arguments are similar to <code>bazel-bin/inception/cifar10_train</code> for multi-gpu mode but with</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="--job_name JOB_NAME One of "ps", "worker" --task_id TASK_ID Task ID of the worker/replica running the training. --ps_hosts PS_HOSTS Comma-separated list of hostname:port for the parameter server jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222' --worker_hosts WORKER_HOSTS Comma-separated list of hostname:port for the worker jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222'"><pre class="notranslate"><code>--job_name JOB_NAME One of "ps", "worker" --task_id TASK_ID Task ID of the worker/replica running the training. --ps_hosts PS_HOSTS Comma-separated list of hostname:port for the parameter server jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222' --worker_hosts WORKER_HOSTS Comma-separated list of hostname:port for the worker jobs. e.g. 'machine1:2222,machine2:1111,machine2:2222' </code></pre></div> <p dir="auto">For more details, type <code>bazel-bin/inception/cifar10_distributed_train --help</code> or go <a href="#backup-inception-in-tensorflow">here</a>.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">A single script to launch all</h2><a id="user-content-a-single-script-to-launch-all" class="anchor" aria-label="Permalink: A single script to launch all" href="#a-single-script-to-launch-all"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Config <a href="/wenwei202/terngrad/blob/master/terngrad/config_dist.sh">config_dist.sh</a> and run <a href="/wenwei202/terngrad/blob/master/terngrad/run_dist.sh">run_dist.sh</a>.</p> <p dir="auto">You only need to configure <code>config_dist.sh</code> (including workers, ps, gpu devices and dataset paths), and write a <code>WORKER_SCRIPT</code> to specify how to start a worker. <a href="/wenwei202/terngrad/blob/master/terngrad/run_single_worker_cifarnet.sh">run_single_worker_cifarnet.sh</a> and <a href="/wenwei202/terngrad/blob/master/terngrad/run_single_worker_alexnet.sh">run_single_worker_alexnet.sh</a> are two <code>WORKER_SCRIPT</code> examples, which basically set hyperparameters and start training.</p> <p dir="auto">Usage is explained within <code>config_dist.sh</code> script.</p> <p dir="auto">By default, results are saved in <code>${HOME}/tmp/</code>.</p> <p dir="auto">We also provide <a href="/wenwei202/terngrad/blob/master/terngrad/split_dataset.sh">split_dataset.sh</a> to <em>locally</em> split shards of training dataset. Usage: <code>./split_dataset.sh <path-of-dataset-to-be-split> <total_workers> <worker_index></code>. It will create a subfolder under <code><path-of-dataset-to-be-split></code> named as <code>worker_<worker_index>_of_<total_workers></code>, and create links to shard files belonging to this worker. For example,</p> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ cd ~/dataset/imagenet-data/ $ ls -l -rw-rw-r-- 1 wew57 wew57 144582762 Mar 24 2017 train-00000-of-01024 -rw-rw-r-- 1 wew57 wew57 148475588 Mar 24 2017 train-00001-of-01024 -rw-rw-r-- 1 wew57 wew57 150196808 Mar 24 2017 train-00002-of-01024 ... -rw-rw-r-- 1 wew57 wew57 144180160 Mar 24 2017 train-01021-of-01024 -rw-rw-r-- 1 wew57 wew57 140903282 Mar 24 2017 train-01022-of-01024 -rw-rw-r-- 1 wew57 wew57 138485470 Mar 24 2017 train-01023-of-01024 $ cd /home/wew57/github/users/wenwei202/terngrad/terngrad $ ./split_dataset.sh ~/dataset/imagenet-data/ 16 1 Splitting to /home/wew57/dataset/imagenet-data//worker_1_of_16 ... $ ls -l /home/wew57/dataset/imagenet-data//worker_1_of_16 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00064-of-01024 -> /home/wew57/dataset/imagenet-data//train-00064-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00065-of-01024 -> /home/wew57/dataset/imagenet-data//train-00065-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00066-of-01024 -> /home/wew57/dataset/imagenet-data//train-00066-of-01024 ... lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00125-of-01024 -> /home/wew57/dataset/imagenet-data//train-00125-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00126-of-01024 -> /home/wew57/dataset/imagenet-data//train-00126-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00127-of-01024 -> /home/wew57/dataset/imagenet-data//train-00127-of-01024"><pre class="notranslate"><code>$ cd ~/dataset/imagenet-data/ $ ls -l -rw-rw-r-- 1 wew57 wew57 144582762 Mar 24 2017 train-00000-of-01024 -rw-rw-r-- 1 wew57 wew57 148475588 Mar 24 2017 train-00001-of-01024 -rw-rw-r-- 1 wew57 wew57 150196808 Mar 24 2017 train-00002-of-01024 ... -rw-rw-r-- 1 wew57 wew57 144180160 Mar 24 2017 train-01021-of-01024 -rw-rw-r-- 1 wew57 wew57 140903282 Mar 24 2017 train-01022-of-01024 -rw-rw-r-- 1 wew57 wew57 138485470 Mar 24 2017 train-01023-of-01024 $ cd /home/wew57/github/users/wenwei202/terngrad/terngrad $ ./split_dataset.sh ~/dataset/imagenet-data/ 16 1 Splitting to /home/wew57/dataset/imagenet-data//worker_1_of_16 ... $ ls -l /home/wew57/dataset/imagenet-data//worker_1_of_16 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00064-of-01024 -> /home/wew57/dataset/imagenet-data//train-00064-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00065-of-01024 -> /home/wew57/dataset/imagenet-data//train-00065-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00066-of-01024 -> /home/wew57/dataset/imagenet-data//train-00066-of-01024 ... lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00125-of-01024 -> /home/wew57/dataset/imagenet-data//train-00125-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00126-of-01024 -> /home/wew57/dataset/imagenet-data//train-00126-of-01024 lrwxrwxrwx 1 wew57 wew57 55 Sep 30 17:30 train-00127-of-01024 -> /home/wew57/dataset/imagenet-data//train-00127-of-01024 </code></pre></div> <p dir="auto">You can stop all tasks by <a href="/wenwei202/terngrad/blob/master/terngrad/stop_dist.sh">stop_dist.sh</a></p> <p dir="auto">Currently, distributed-node mode only supports 32bit gradients. It will take a while to hack the highly-encapsulated <code>SyncReplicasOptimizer</code> to integrate TernGrad. Keep updating.</p> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Python executables</h1><a id="user-content-python-executables" class="anchor" aria-label="Permalink: Python executables" href="#python-executables"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Bash scripts essentially call python executables. We list python commands here for agile development. Taking 32bit gradients as examples.</p> <p dir="auto">Node that, in TernGrad, parameters are allocated in each GPU to reduce communication because we can communicate quantized gradients instead of floating parameters. By default, the program saves parameters in all GPUs. To evaluate/test, use <code>--tower <gpu_id></code> to specify which GPU's parameters you want to test on. (We will try to remove this feature to save storage, because the parameter sets are identical across all GPUs). Alteratively, you can use <code>--save_tower 0</code> in training executables to avoid saving duplicated parameters, in which case, <code>--tower <gpu_id></code> is unnecessary during evaluation/testing.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Build and run evaluating/training LeNet on mnist</h2><a id="user-content-build-and-run-evaluatingtraining-lenet-on-mnist" class="anchor" aria-label="Permalink: Build and run evaluating/training LeNet on mnist" href="#build-and-run-evaluatingtraining-lenet-on-mnist"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/terngrad bazel build inception/mnist_train bazel build inception/mnist_eval bazel-bin/inception/mnist_train \ --optimizer momentum \ --initial_learning_rate 0.01 \ --learning_rate_decay_type polynomial \ --max_steps 10000 \ --net lenet \ --image_size 28 \ --num_gpus 2 \ --batch_size 64 \ --train_dir /tmp/mnist_train \ --data_dir ~/dataset/mnist-data/ bazel-bin/inception/mnist_eval \ --data_dir ~/dataset/mnist-data/ \ --net lenet \ --image_size 28 \ --batch_size 100 \ --checkpoint_dir /tmp/mnist_train \ --restore_avg_var True \ --eval_interval_secs 300 \ --eval_dir /tmp/mnist_eval \ --subset test"><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/terngrad bazel build inception/mnist_train bazel build inception/mnist_eval bazel-bin/inception/mnist_train \ --optimizer momentum \ --initial_learning_rate 0.01 \ --learning_rate_decay_type polynomial \ --max_steps 10000 \ --net lenet \ --image_size 28 \ --num_gpus 2 \ --batch_size 64 \ --train_dir /tmp/mnist_train \ --data_dir ~/dataset/mnist-data/ bazel-bin/inception/mnist_eval \ --data_dir ~/dataset/mnist-data/ \ --net lenet \ --image_size 28 \ --batch_size 100 \ --checkpoint_dir /tmp/mnist_train \ --restore_avg_var True \ --eval_interval_secs 300 \ --eval_dir /tmp/mnist_eval \ --subset test </code></pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Build and run evaluating/training on cifar-10</h2><a id="user-content-build-and-run-evaluatingtraining-on-cifar-10" class="anchor" aria-label="Permalink: Build and run evaluating/training on cifar-10" href="#build-and-run-evaluatingtraining-on-cifar-10"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/terngrad bazel build inception/cifar10_train bazel build inception/cifar10_eval bazel-bin/inception/cifar10_train \ --optimizer adam \ --initial_learning_rate 0.0002 \ --num_epochs_per_decay 256 \ --max_steps 200000 \ --net cifar10_alexnet \ --image_size 24 \ --num_gpus 2 \ --batch_size 128 \ --train_dir /tmp/cifar10_train \ --data_dir ~/dataset/cifar10-data/ bazel-bin/inception/cifar10_eval \ --data_dir ~/dataset/cifar10-data/ \ --net cifar10_alexnet \ --image_size 24 \ --batch_size 50 \ --checkpoint_dir /tmp/cifar10_train \ --restore_avg_var True \ --eval_interval_secs 300 \ --eval_dir /tmp/cifar10_eval \ --subset test "><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/terngrad bazel build inception/cifar10_train bazel build inception/cifar10_eval bazel-bin/inception/cifar10_train \ --optimizer adam \ --initial_learning_rate 0.0002 \ --num_epochs_per_decay 256 \ --max_steps 200000 \ --net cifar10_alexnet \ --image_size 24 \ --num_gpus 2 \ --batch_size 128 \ --train_dir /tmp/cifar10_train \ --data_dir ~/dataset/cifar10-data/ bazel-bin/inception/cifar10_eval \ --data_dir ~/dataset/cifar10-data/ \ --net cifar10_alexnet \ --image_size 24 \ --batch_size 50 \ --checkpoint_dir /tmp/cifar10_train \ --restore_avg_var True \ --eval_interval_secs 300 \ --eval_dir /tmp/cifar10_eval \ --subset test </code></pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Build and run ImageNet</h2><a id="user-content-build-and-run-imagenet" class="anchor" aria-label="Permalink: Build and run ImageNet" href="#build-and-run-imagenet"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd ${TERNGRAD_ROOT}/terngrad bazel build inception/imagenet_train bazel build inception/imagenet_eval bazel-bin/inception/imagenet_train \ --optimizer momentum \ --net alexnet \ --image_size 224 \ --num_gpus 2 \ --batch_size 256 \ --train_dir /tmp/imagenet_train \ --data_dir ~/dataset/imagenet-data/ bazel-bin/inception/imagenet_eval \ --data_dir ~/dataset/imagenet-data/ \ --net alexnet \ --image_size 224 \ --batch_size 50 \ --checkpoint_dir /tmp/imagenet_train \ --restore_avg_var True \ --eval_dir /tmp/imagenet_eval "><pre class="notranslate"><code>cd ${TERNGRAD_ROOT}/terngrad bazel build inception/imagenet_train bazel build inception/imagenet_eval bazel-bin/inception/imagenet_train \ --optimizer momentum \ --net alexnet \ --image_size 224 \ --num_gpus 2 \ --batch_size 256 \ --train_dir /tmp/imagenet_train \ --data_dir ~/dataset/imagenet-data/ bazel-bin/inception/imagenet_eval \ --data_dir ~/dataset/imagenet-data/ \ --net alexnet \ --image_size 224 \ --batch_size 50 \ --checkpoint_dir /tmp/imagenet_train \ --restore_avg_var True \ --eval_dir /tmp/imagenet_eval </code></pre></div> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Open questions</h1><a id="user-content-open-questions" class="anchor" aria-label="Permalink: Open questions" href="#open-questions"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <ol dir="auto"> <li>How will TernGrad work with asynchronous SGD</li> <li>How to reduce variance of TernGrad when the larger variance introduces some accuracy loss</li> <li>How will TernGrad work when server-to-worker gradients are ternarized in the same way</li> </ol> <div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">Backup (Inception in TensorFlow)</h1><a id="user-content-backup-inception-in-tensorflow" class="anchor" aria-label="Permalink: Backup (Inception in TensorFlow)" href="#backup-inception-in-tensorflow"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto"><a href="http://www.image-net.org/" rel="nofollow">ImageNet</a> is a common academic data set in machine learning for training an image recognition system. Code in this directory demonstrates how to use TensorFlow to train and evaluate a type of convolutional neural network (CNN) on this academic data set. In particular, we demonstrate how to train the Inception v3 architecture as specified in:</p> <p dir="auto"><em>Rethinking the Inception Architecture for Computer Vision</em></p> <p dir="auto">Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, Zbigniew Wojna</p> <p dir="auto"><a href="http://arxiv.org/abs/1512.00567" rel="nofollow">http://arxiv.org/abs/1512.00567</a></p> <p dir="auto">This network achieves 21.2% top-1 and 5.6% top-5 error for single frame evaluation with a computational cost of 5 billion multiply-adds per inference and with using less than 25 million parameters. Below is a visualization of the model architecture.</p>  <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Description of Code</h2><a id="user-content-description-of-code" class="anchor" aria-label="Permalink: Description of Code" href="#description-of-code"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">The code base provides three core binaries for:</p> <ul dir="auto"> <li>Training an Inception v3 network from scratch across multiple GPUs and/or multiple machines using the ImageNet 2012 Challenge training data set.</li> <li>Evaluating an Inception v3 network using the ImageNet 2012 Challenge validation data set.</li> <li>Retraining an Inception v3 network on a novel task and back-propagating the errors to fine tune the network weights.</li> </ul> <p dir="auto">The training procedure employs synchronous stochastic gradient descent across multiple GPUs. The user may specify the number of GPUs they wish harness. The synchronous training performs <em>batch-splitting</em> by dividing a given batch across multiple GPUs.</p> <p dir="auto">The training set up is nearly identical to the section [Training a Model Using Multiple GPU Cards] (<a href="https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards" rel="nofollow">https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards</a>) where we have substituted the CIFAR-10 model architecture with Inception v3. The primary differences with that setup are:</p> <ul dir="auto"> <li>Calculate and update the batch-norm statistics during training so that they may be substituted in during evaluation.</li> <li>Specify the model architecture using a (still experimental) higher level language called TensorFlow-Slim.</li> </ul> <p dir="auto">For more details about TensorFlow-Slim, please see the [Slim README] (inception/slim/README.md). Please note that this higher-level language is still <em>experimental</em> and the API may change over time depending on usage and subsequent research.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Getting Started</h2><a id="user-content-getting-started" class="anchor" aria-label="Permalink: Getting Started" href="#getting-started"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto"><strong>NOTE</strong> Before doing anything, we first need to build TensorFlow from source, and installed as a PIP package. Please follow the instructions at [Installing From Source] (<a href="https://www.tensorflow.org/get_started/os_setup.html#create-the-pip-package-and-install" rel="nofollow">https://www.tensorflow.org/get_started/os_setup.html#create-the-pip-package-and-install</a>).</p> <p dir="auto">Before you run the training script for the first time, you will need to download and convert the ImageNet data to native TFRecord format. The TFRecord format consists of a set of sharded files where each entry is a serialized <code>tf.Example</code> proto. Each <code>tf.Example</code> proto contains the ImageNet image (JPEG encoded) as well as metadata such as label and bounding box information. See <a href="/wenwei202/terngrad/blob/master/inception/image_processing.py"><code>parse_example_proto</code></a> for details.</p> <p dir="auto">We provide a single <a href="/wenwei202/terngrad/blob/master/inception/data/download_and_preprocess_imagenet.sh">script</a> for downloading and converting ImageNet data to TFRecord format. Downloading and preprocessing the data may take several hours (up to half a day) depending on your network and computer speed. Please be patient.</p> <p dir="auto">To begin, you will need to sign up for an account with [ImageNet] (<a href="http://image-net.org" rel="nofollow">http://image-net.org</a>) to gain access to the data. Look for the sign up page, create an account and request an access key to download the data.</p> <p dir="auto">After you have <code>USERNAME</code> and <code>PASSWORD</code>, you are ready to run our script. Make sure that your hard disk has at least 500 GB of free space for downloading and storing the data. Here we select <code>DATA_DIR=$HOME/dataset/imagenet-data</code> as such a location but feel free to edit accordingly.</p> <p dir="auto">When you run the below script, please enter <em>USERNAME</em> and <em>PASSWORD</em> when prompted. This will occur at the very beginning. Once these values are entered, you will not need to interact with the script again.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# location of where to place the ImageNet data DATA_DIR=$HOME/dataset/imagenet-data # build the preprocessing script. bazel build inception/download_and_preprocess_imagenet # run it bazel-bin/inception/download_and_preprocess_imagenet "${DATA_DIR}""><pre><span class="pl-c"><span class="pl-c">#</span> location of where to place the ImageNet data</span> DATA_DIR=<span class="pl-smi">$HOME</span>/dataset/imagenet-data <span class="pl-c"><span class="pl-c">#</span> build the preprocessing script.</span> bazel build inception/download_and_preprocess_imagenet <span class="pl-c"><span class="pl-c">#</span> run it</span> bazel-bin/inception/download_and_preprocess_imagenet <span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${DATA_DIR}</span><span class="pl-pds">"</span></span></pre></div> <p dir="auto">The final line of the output script should read:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="2016-02-17 14:30:17.287989: Finished writing all 1281167 images in data set."><pre>2016-02-17 14:30:17.287989: Finished writing all 1281167 images <span class="pl-k">in</span> data set.</pre></div> <p dir="auto">When the script finishes you will find 1024 and 128 training and validation files in the <code>DATA_DIR</code>. The files will match the patterns <code>train-????-of-1024</code> and <code>validation-?????-of-00128</code>, respectively.</p> <p dir="auto"><a href="https://www.youtube.com/watch?v=9bZkp7q19f0" rel="nofollow">Congratulations!</a> You are now ready to train or evaluate with the ImageNet data set.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Train from Scratch</h2><a id="user-content-how-to-train-from-scratch" class="anchor" aria-label="Permalink: How to Train from Scratch" href="#how-to-train-from-scratch"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto"><strong>WARNING</strong> Training an Inception v3 network from scratch is a computationally intensive task and depending on your compute setup may take several days or even weeks.</p> <p dir="auto"><em>Before proceeding</em> please read the [Convolutional Neural Networks] (<a href="https://www.tensorflow.org/tutorials/deep_cnn/index.html" rel="nofollow">https://www.tensorflow.org/tutorials/deep_cnn/index.html</a>) tutorial in particular focus on [Training a Model Using Multiple GPU Cards] (<a href="https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards" rel="nofollow">https://www.tensorflow.org/tutorials/deep_cnn/index.html#training-a-model-using-multiple-gpu-cards</a>) . The model training method is nearly identical to that described in the CIFAR-10 multi-GPU model training. Briefly, the model training</p> <ul dir="auto"> <li>Places an individual model replica on each GPU. Split the batch across the GPUs.</li> <li>Updates model parameters synchronously by waiting for all GPUs to finish processing a batch of data.</li> </ul> <p dir="auto">The training procedure is encapsulated by this diagram of how operations and variables are placed on CPU and GPUs respectively.</p> <div dir="auto"> <a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/bc2f3dcd0ff224146babb58aff157064dccf7eb320a72533a52f0c7d2981b3fb/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f506172616c6c656c69736d2e706e67"><img style="width: 100%; max-width: 100%;" src="https://camo.githubusercontent.com/bc2f3dcd0ff224146babb58aff157064dccf7eb320a72533a52f0c7d2981b3fb/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f506172616c6c656c69736d2e706e67" data-canonical-src="https://www.tensorflow.org/images/Parallelism.png"></a> </div> <p dir="auto">Each tower computes the gradients for a portion of the batch and the gradients are combined and averaged across the multiple towers in order to provide a single update of the Variables stored on the CPU.</p> <p dir="auto">A crucial aspect of training a network of this size is <em>training speed</em> in terms of wall-clock time. The training speed is dictated by many factors -- most importantly the batch size and the learning rate schedule. Both of these parameters are heavily coupled to the hardware set up.</p> <p dir="auto">Generally speaking, a batch size is a difficult parameter to tune as it requires balancing memory demands of the model, memory available on the GPU and speed of computation. Generally speaking, employing larger batch sizes leads to more efficient computation and potentially more efficient training steps.</p> <p dir="auto">We have tested several hardware setups for training this model from scratch but we emphasize that depending your hardware set up, you may need to adapt the batch size and learning rate schedule.</p> <p dir="auto">Please see the comments in <code>inception_train.py</code> for a few selected learning rate plans based on some selected hardware setups.</p> <p dir="auto">To train this model, you simply need to specify the following:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/imagenet_train # run it bazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/tmp/imagenet_train --data_dir=/tmp/imagenet_data"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/imagenet_train <span class="pl-c"><span class="pl-c">#</span> run it</span> bazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/tmp/imagenet_train --data_dir=/tmp/imagenet_data</pre></div> <p dir="auto">The model reads in the ImageNet training data from <code>--data_dir</code>. If you followed the instructions in <a href="#getting-started">Getting Started</a>, then set <code>--data_dir="${DATA_DIR}"</code>. The script assumes that there exists a set of sharded TFRecord files containing the ImageNet data. If you have not created TFRecord files, please refer to <a href="#getting-started">Getting Started</a></p> <p dir="auto">Here is the output of the above command line when running on a Tesla K40c:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="2016-03-07 12:24:59.922898: step 0, loss = 13.11 (5.3 examples/sec; 6.064 sec/batch) 2016-03-07 12:25:55.206783: step 10, loss = 13.71 (9.4 examples/sec; 3.394 sec/batch) 2016-03-07 12:26:28.905231: step 20, loss = 14.81 (9.5 examples/sec; 3.380 sec/batch) 2016-03-07 12:27:02.699719: step 30, loss = 14.45 (9.5 examples/sec; 3.378 sec/batch) 2016-03-07 12:27:36.515699: step 40, loss = 13.98 (9.5 examples/sec; 3.376 sec/batch) 2016-03-07 12:28:10.220956: step 50, loss = 13.92 (9.6 examples/sec; 3.327 sec/batch) 2016-03-07 12:28:43.658223: step 60, loss = 13.28 (9.6 examples/sec; 3.350 sec/batch) ..."><pre>2016-03-07 12:24:59.922898: step 0, loss = 13.11 (5.3 examples/sec<span class="pl-k">;</span> 6.064 sec/batch) 2016-03-07 12:25:55.206783: step 10, loss = 13.71 (9.4 examples/sec<span class="pl-k">;</span> 3.394 sec/batch) 2016-03-07 12:26:28.905231: step 20, loss = 14.81 (9.5 examples/sec<span class="pl-k">;</span> 3.380 sec/batch) 2016-03-07 12:27:02.699719: step 30, loss = 14.45 (9.5 examples/sec<span class="pl-k">;</span> 3.378 sec/batch) 2016-03-07 12:27:36.515699: step 40, loss = 13.98 (9.5 examples/sec<span class="pl-k">;</span> 3.376 sec/batch) 2016-03-07 12:28:10.220956: step 50, loss = 13.92 (9.6 examples/sec<span class="pl-k">;</span> 3.327 sec/batch) 2016-03-07 12:28:43.658223: step 60, loss = 13.28 (9.6 examples/sec<span class="pl-k">;</span> 3.350 sec/batch) ...</pre></div> <p dir="auto">In this example, a log entry is printed every 10 step and the line includes the total loss (starts around 13.0-14.0) and the speed of processing in terms of throughput (examples / sec) and batch speed (sec/batch).</p> <p dir="auto">The number of GPU devices is specified by <code>--num_gpus</code> (which defaults to 1). Specifying <code>--num_gpus</code> greater then 1 splits the batch evenly split across the GPU cards.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/imagenet_train # run it bazel-bin/inception/imagenet_train --num_gpus=2 --batch_size=64 --train_dir=/tmp/imagenet_train"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/imagenet_train <span class="pl-c"><span class="pl-c">#</span> run it</span> bazel-bin/inception/imagenet_train --num_gpus=2 --batch_size=64 --train_dir=/tmp/imagenet_train</pre></div> <p dir="auto">This model splits the batch of 64 images across 2 GPUs and calculates the average gradient by waiting for both GPUs to finish calculating the gradients from their respective data (See diagram above). Generally speaking, using larger numbers of GPUs leads to higher throughput as well as the opportunity to use larger batch sizes. In turn, larger batch sizes imply better estimates of the gradient enabling the usage of higher learning rates. In summary, using more GPUs results in simply faster training speed.</p> <p dir="auto">Note that selecting a batch size is a difficult parameter to tune as it requires balancing memory demands of the model, memory available on the GPU and speed of computation. Generally speaking, employing larger batch sizes leads to more efficient computation and potentially more efficient training steps.</p> <p dir="auto">Note that there is considerable noise in the loss function on individual steps in the previous log. Because of this noise, it is difficult to discern how well a model is learning. The solution to the last problem is to launch TensorBoard pointing to the directory containing the events log.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="tensorboard --logdir=/tmp/imagenet_train"><pre>tensorboard --logdir=/tmp/imagenet_train</pre></div> <p dir="auto">TensorBoard has access to the many Summaries produced by the model that describe multitudes of statistics tracking the model behavior and the quality of the learned model. In particular, TensorBoard tracks a exponentially smoothed version of the loss. In practice, it is far easier to judge how well a model learns by monitoring the smoothed version of the loss.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Train from Scratch in a Distributed Setting</h2><a id="user-content-how-to-train-from-scratch-in-a-distributed-setting" class="anchor" aria-label="Permalink: How to Train from Scratch in a Distributed Setting" href="#how-to-train-from-scratch-in-a-distributed-setting"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto"><strong>NOTE</strong> Distributed TensorFlow requires version 0.8 or later.</p> <p dir="auto">Distributed TensorFlow lets us use multiple machines to train a model faster. This is quite different from the training with multiple GPU towers on a single machine where all parameters and gradients computation are in the same place. We coordinate the computation across multiple machines by employing a centralized repository for parameters that maintains a unified, single copy of model parameters. Each individual machine sends gradient updates to the centralized parameter repository which coordinates these updates and sends back updated parameters to the individual machines running the model training.</p> <p dir="auto">We term each machine that runs a copy of the training a <code>worker</code> or <code>replica</code>. We term each machine that maintains model parameters a <code>ps</code>, short for <code>parameter server</code>. Note that we might have more than one machine acting as a <code>ps</code> as the model parameters may be sharded across multiple machines.</p> <p dir="auto">Variables may be updated with synchronous or asynchronous gradient updates. One may construct a an [<code>Optimizer</code>] (<a href="https://www.tensorflow.org/api_docs/python/train.html#optimizers" rel="nofollow">https://www.tensorflow.org/api_docs/python/train.html#optimizers</a>) in TensorFlow that constructs the necessary graph for either case diagrammed below from TensorFlow [Whitepaper] (<a href="http://download.tensorflow.org/paper/whitepaper2015.pdf" rel="nofollow">http://download.tensorflow.org/paper/whitepaper2015.pdf</a>):</p> <div dir="auto"> <a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/44e2ae57208da9a61ccb7d87bf0491a5ba6806f16e97e2511e57abe6e7e06e27/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f74656e736f72666c6f775f666967757265372e706e67"><img style="width: 100%; max-width: 100%;" src="https://camo.githubusercontent.com/44e2ae57208da9a61ccb7d87bf0491a5ba6806f16e97e2511e57abe6e7e06e27/68747470733a2f2f7777772e74656e736f72666c6f772e6f72672f696d616765732f74656e736f72666c6f775f666967757265372e706e67" data-canonical-src="https://www.tensorflow.org/images/tensorflow_figure7.png"></a> </div> <p dir="auto">In <a href="https://arxiv.org/abs/1604.00981" rel="nofollow">a recent paper</a>, synchronous gradient updates have demonstrated to reach higher accuracy in a shorter amount of time. In this distributed Inception example we employ synchronous gradient updates.</p> <p dir="auto">Note that in this example each replica has a single tower that uses one GPU.</p> <p dir="auto">The command-line flags <code>worker_hosts</code> and <code>ps_hosts</code> specify available servers. The same binary will be used for both the <code>worker</code> jobs and the <code>ps</code> jobs. Command line flag <code>job_name</code> will be used to specify what role a task will be playing and <code>task_id</code> will be used to idenify which one of the jobs it is running. Several things to note here:</p> <ul dir="auto"> <li>The numbers of <code>ps</code> and <code>worker</code> tasks are inferred from the lists of hosts specified in the flags. The <code>task_id</code> should be within the range <code>[0, num_ps_tasks)</code> for <code>ps</code> tasks and <code>[0, num_worker_tasks)</code> for <code>worker</code> tasks.</li> <li><code>ps</code> and <code>worker</code> tasks can run on the same machine, as long as that machine has sufficient resources to handle both tasks. Note that the <code>ps</code> task does not benefit from a GPU, so it should not attempt to use one (see below).</li> <li>Multiple <code>worker</code> tasks can run on the same machine with multiple GPUs so machine_A with 2 GPUs may have 2 workers while machine_B with 1 GPU just has 1 worker.</li> <li>The default learning rate schedule works well for a wide range of number of replicas [25, 50, 100] but feel free to tune it for even better results.</li> <li>The command line of both <code>ps</code> and <code>worker</code> tasks should include the complete list of <code>ps_hosts</code> and <code>worker_hosts</code>.</li> <li>There is a chief <code>worker</code> among all workers which defaults to <code>worker</code> 0. The chief will be in charge of initializing all the parameters, writing out the summaries and the checkpoint. The checkpoint and summary will be in the <code>train_dir</code> of the host for <code>worker</code> 0.</li> <li>Each worker processes a batch_size number of examples but each gradient update is computed from all replicas. Hence, the effective batch size of this model is batch_size * num_workers.</li> </ul> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/imagenet_distributed_train # To start worker 0, go to the worker0 host and run the following (Note that # task_id should be in the range [0, num_worker_tasks): bazel-bin/inception/imagenet_distributed_train \ --batch_size=32 \ --data_dir=$HOME/imagenet-data \ --job_name='worker' \ --task_id=0 \ --ps_hosts='ps0.example.com:2222' \ --worker_hosts='worker0.example.com:2222,worker1.example.com:2222' # To start worker 1, go to the worker1 host and run the following (Note that # task_id should be in the range [0, num_worker_tasks): bazel-bin/inception/imagenet_distributed_train \ --batch_size=32 \ --data_dir=$HOME/imagenet-data \ --job_name='worker' \ --task_id=1 \ --ps_hosts='ps0.example.com:2222' \ --worker_hosts='worker0.example.com:2222,worker1.example.com:2222' # To start the parameter server (ps), go to the ps host and run the following (Note # that task_id should be in the range [0, num_ps_tasks): bazel-bin/inception/imagenet_distributed_train \ --job_name='ps' \ --task_id=0 \ --ps_hosts='ps0.example.com:2222' \ --worker_hosts='worker0.example.com:2222,worker1.example.com:2222'"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/imagenet_distributed_train <span class="pl-c"><span class="pl-c">#</span> To start worker 0, go to the worker0 host and run the following (Note that</span> <span class="pl-c"><span class="pl-c">#</span> task_id should be in the range [0, num_worker_tasks):</span> bazel-bin/inception/imagenet_distributed_train \ --batch_size=32 \ --data_dir=<span class="pl-smi">$HOME</span>/imagenet-data \ --job_name=<span class="pl-s"><span class="pl-pds">'</span>worker<span class="pl-pds">'</span></span> \ --task_id=0 \ --ps_hosts=<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span> \ --worker_hosts=<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222,worker1.example.com:2222<span class="pl-pds">'</span></span> <span class="pl-c"><span class="pl-c">#</span> To start worker 1, go to the worker1 host and run the following (Note that</span> <span class="pl-c"><span class="pl-c">#</span> task_id should be in the range [0, num_worker_tasks):</span> bazel-bin/inception/imagenet_distributed_train \ --batch_size=32 \ --data_dir=<span class="pl-smi">$HOME</span>/imagenet-data \ --job_name=<span class="pl-s"><span class="pl-pds">'</span>worker<span class="pl-pds">'</span></span> \ --task_id=1 \ --ps_hosts=<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span> \ --worker_hosts=<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222,worker1.example.com:2222<span class="pl-pds">'</span></span> <span class="pl-c"><span class="pl-c">#</span> To start the parameter server (ps), go to the ps host and run the following (Note</span> <span class="pl-c"><span class="pl-c">#</span> that task_id should be in the range [0, num_ps_tasks):</span> bazel-bin/inception/imagenet_distributed_train \ --job_name=<span class="pl-s"><span class="pl-pds">'</span>ps<span class="pl-pds">'</span></span> \ --task_id=0 \ --ps_hosts=<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span> \ --worker_hosts=<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222,worker1.example.com:2222<span class="pl-pds">'</span></span></pre></div> <p dir="auto">If you have installed a GPU-compatible version of TensorFlow, the <code>ps</code> will also try to allocate GPU memory although it is not helpful. This could potentially crash the worker on the same machine as it has little to no GPU memory to allocate. To avoid this, you can prepend the previous command to start <code>ps</code> with: <code>CUDA_VISIBLE_DEVICES=''</code></p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="CUDA_VISIBLE_DEVICES='' bazel-bin/inception/imagenet_distributed_train \ --job_name='ps' \ --task_id=0 \ --ps_hosts='ps0.example.com:2222' \ --worker_hosts='worker0.example.com:2222,worker1.example.com:2222'"><pre>CUDA_VISIBLE_DEVICES=<span class="pl-s"><span class="pl-pds">'</span><span class="pl-pds">'</span></span> bazel-bin/inception/imagenet_distributed_train \ --job_name=<span class="pl-s"><span class="pl-pds">'</span>ps<span class="pl-pds">'</span></span> \ --task_id=0 \ --ps_hosts=<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span> \ --worker_hosts=<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222,worker1.example.com:2222<span class="pl-pds">'</span></span></pre></div> <p dir="auto">If you have run everything correctly, you should see a log in each <code>worker</code> job that looks like the following. Note the training speed varies depending on your hardware and the first several steps could take much longer.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222'] INFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222'] I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -> {ps0.example.com:2222, ps1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -> {localhost:2222, worker1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222 INFO:tensorflow:Created variable global_step:0 with shape () and init <function zeros_initializer at 0x7f6aa014b140> ... INFO:tensorflow:Created variable logits/logits/biases:0 with shape (1001,) and init <function _initializer at 0x7f6a77f3cf50> INFO:tensorflow:SyncReplicas enabled: replicas_to_aggregate=2; total_num_replicas=2 INFO:tensorflow:2016-04-13 01:56:26.405639 Supervisor INFO:tensorflow:Started 2 queues for processing input data. INFO:tensorflow:global_step/sec: 0 INFO:tensorflow:Worker 0: 2016-04-13 01:58:40.342404: step 0, loss = 12.97(0.0 examples/sec; 65.428 sec/batch) INFO:tensorflow:global_step/sec: 0.0172907 ..."><pre>INFO:tensorflow:PS hosts are: [<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>ps1.example.com:2222<span class="pl-pds">'</span></span>] INFO:tensorflow:Worker hosts are: [<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>worker1.example.com:2222<span class="pl-pds">'</span></span>] I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache <span class="pl-k">for</span> job ps -<span class="pl-k">></span> {ps0.example.com:2222, ps1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache <span class="pl-k">for</span> job worker -<span class="pl-k">></span> {localhost:2222, worker1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222 INFO:tensorflow:Created variable global_step:0 with <span class="pl-en">shape</span> () and init <span class="pl-k"><</span>function zeros_initializer at 0x7f6aa014b<span class="pl-k">140></span> ... INFO:tensorflow:Created variable logits/logits/biases:0 with shape (1001,) and init <span class="pl-k"><</span>function _initializer at 0x7f6a77f3cf<span class="pl-k">50></span> INFO:tensorflow:SyncReplicas enabled: replicas_to_aggregate=2<span class="pl-k">;</span> total_num_replicas=2 INFO:tensorflow:2016-04-13 01:56:26.405639 Supervisor INFO:tensorflow:Started 2 queues <span class="pl-k">for</span> processing input data. INFO:tensorflow:global_step/sec: 0 INFO:tensorflow:Worker 0: 2016-04-13 01:58:40.342404: step 0, loss = 12.97(0.0 examples/sec<span class="pl-k">;</span> 65.428 sec/batch) INFO:tensorflow:global_step/sec: 0.0172907 ...</pre></div> <p dir="auto">and a log in each <code>ps</code> job that looks like the following:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="INFO:tensorflow:PS hosts are: ['ps0.example.com:2222', 'ps1.example.com:2222'] INFO:tensorflow:Worker hosts are: ['worker0.example.com:2222', 'worker1.example.com:2222'] I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job ps -> {localhost:2222, ps1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache for job worker -> {worker0.example.com:2222, worker1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222"><pre>INFO:tensorflow:PS hosts are: [<span class="pl-s"><span class="pl-pds">'</span>ps0.example.com:2222<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>ps1.example.com:2222<span class="pl-pds">'</span></span>] INFO:tensorflow:Worker hosts are: [<span class="pl-s"><span class="pl-pds">'</span>worker0.example.com:2222<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>worker1.example.com:2222<span class="pl-pds">'</span></span>] I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache <span class="pl-k">for</span> job ps -<span class="pl-k">></span> {localhost:2222, ps1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:206] Initialize HostPortsGrpcChannelCache <span class="pl-k">for</span> job worker -<span class="pl-k">></span> {worker0.example.com:2222, worker1.example.com:2222} I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:202] Started server with target: grpc://localhost:2222</pre></div> <p dir="auto"><a href="https://www.youtube.com/watch?v=9bZkp7q19f0" rel="nofollow">Congratulations!</a> You are now training Inception in a distributed manner.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Evaluate</h2><a id="user-content-how-to-evaluate" class="anchor" aria-label="Permalink: How to Evaluate" href="#how-to-evaluate"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Evaluating an Inception v3 model on the ImageNet 2012 validation data set requires running a separate binary.</p> <p dir="auto">The evaluation procedure is nearly identical to [Evaluating a Model] (<a href="https://www.tensorflow.org/tutorials/deep_cnn/index.html#evaluating-a-model" rel="nofollow">https://www.tensorflow.org/tutorials/deep_cnn/index.html#evaluating-a-model</a>) described in the [Convolutional Neural Network] (<a href="https://www.tensorflow.org/tutorials/deep_cnn/index.html" rel="nofollow">https://www.tensorflow.org/tutorials/deep_cnn/index.html</a>) tutorial.</p> <p dir="auto"><strong>WARNING</strong> Be careful not to run the evaluation and training binary on the same GPU or else you might run out of memory. Consider running the evaluation on a separate GPU if available or suspending the training binary while running the evaluation on the same GPU.</p> <p dir="auto">Briefly, one can evaluate the model by running:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/imagenet_eval # run it bazel-bin/inception/imagenet_eval --checkpoint_dir=/tmp/imagenet_train --eval_dir=/tmp/imagenet_eval"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/imagenet_eval <span class="pl-c"><span class="pl-c">#</span> run it</span> bazel-bin/inception/imagenet_eval --checkpoint_dir=/tmp/imagenet_train --eval_dir=/tmp/imagenet_eval</pre></div> <p dir="auto">Note that we point <code>--checkpoint_dir</code> to the location of the checkpoints saved by <code>inception_train.py</code> above. Running the above command results in the following output:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="2016-02-17 22:32:50.391206: precision @ 1 = 0.735 ..."><pre>2016-02-17 22:32:50.391206: precision @ 1 = 0.735 ...</pre></div> <p dir="auto">The script calculates the precision @ 1 over the entire validation data periodically. The precision @ 1 measures the how often the highest scoring prediction from the model matched the ImageNet label -- in this case, 73.5%. If you wish to run the eval just once and not periodically, append the <code>--run_once</code> option.</p> <p dir="auto">Much like the training script, <code>imagenet_eval.py</code> also exports summaries that may be visualized in TensorBoard. These summaries calculate additional statistics on the predictions (e.g. recall @ 5) as well as monitor the statistics of the model activations and weights during evaluation.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Fine-Tune a Pre-Trained Model on a New Task</h2><a id="user-content-how-to-fine-tune-a-pre-trained-model-on-a-new-task" class="anchor" aria-label="Permalink: How to Fine-Tune a Pre-Trained Model on a New Task" href="#how-to-fine-tune-a-pre-trained-model-on-a-new-task"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Getting Started</h3><a id="user-content-getting-started-1" class="anchor" aria-label="Permalink: Getting Started" href="#getting-started-1"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Much like training the ImageNet model we must first convert a new data set to the sharded TFRecord format which each entry is a serialized <code>tf.Example</code> proto.</p> <p dir="auto">We have provided a script demonstrating how to do this for small data set of of a few thousand flower images spread across 5 labels:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="daisy, dandelion, roses, sunflowers, tulips"><pre>daisy, dandelion, roses, sunflowers, tulips</pre></div> <p dir="auto">There is a single automated script that downloads the data set and converts it to the TFRecord format. Much like the ImageNet data set, each record in the TFRecord format is a serialized <code>tf.Example</code> proto whose entries include a JPEG-encoded string and an integer label. Please see [<code>parse_example_proto</code>] (inception/image_processing.py) for details.</p> <p dir="auto">The script just takes a few minutes to run depending your network connection speed for downloading and processing the images. Your hard disk requires 200MB of free storage. Here we select <code>DATA_DIR=$HOME/flowers-data</code> as such a location but feel free to edit accordingly.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# location of where to place the flowers data FLOWERS_DATA_DIR=$HOME/flowers-data # build the preprocessing script. bazel build inception/download_and_preprocess_flowers # run it bazel-bin/inception/download_and_preprocess_flowers "${FLOWERS_DATA_DIR}""><pre><span class="pl-c"><span class="pl-c">#</span> location of where to place the flowers data</span> FLOWERS_DATA_DIR=<span class="pl-smi">$HOME</span>/flowers-data <span class="pl-c"><span class="pl-c">#</span> build the preprocessing script.</span> bazel build inception/download_and_preprocess_flowers <span class="pl-c"><span class="pl-c">#</span> run it</span> bazel-bin/inception/download_and_preprocess_flowers <span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${FLOWERS_DATA_DIR}</span><span class="pl-pds">"</span></span></pre></div> <p dir="auto">If the script runs successfully, the final line of the terminal output should look like:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="2016-02-24 20:42:25.067551: Finished writing all 3170 images in data set."><pre>2016-02-24 20:42:25.067551: Finished writing all 3170 images <span class="pl-k">in</span> data set.</pre></div> <p dir="auto">When the script finishes you will find 2 shards for the training and validation files in the <code>DATA_DIR</code>. The files will match the patterns <code>train-????-of-00001</code> and <code>validation-?????-of-00001</code>, respectively.</p> <p dir="auto"><strong>NOTE</strong> If you wish to prepare a custom image data set for transfer learning, you will need to invoke <a href="/wenwei202/terngrad/blob/master/inception/data/build_image_data.py"><code>build_image_data.py</code></a> on your custom data set. Please see the associated options and assumptions behind this script by reading the comments section of [<code>build_image_data.py</code>] (inception/data/build_image_data.py). Also, if your custom data has a different number of examples or classes, you need to change the appropriate values in <a href="/wenwei202/terngrad/blob/master/inception/imagenet_data.py"><code>imagenet_data.py</code></a>.</p> <p dir="auto">The second piece you will need is a trained Inception v3 image model. You have the option of either training one yourself (See [How to Train from Scratch] (#how-to-train-from-scratch) for details) or you can download a pre-trained model like so:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# location of where to place the Inception v3 model DATA_DIR=$HOME/inception-v3-model cd ${DATA_DIR} # download the Inception v3 model curl -O http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz tar xzf inception-v3-2016-03-01.tar.gz # this will create a directory called inception-v3 which contains the following files. > ls inception-v3 README.txt checkpoint model.ckpt-157585"><pre><span class="pl-c"><span class="pl-c">#</span> location of where to place the Inception v3 model</span> DATA_DIR=<span class="pl-smi">$HOME</span>/inception-v3-model <span class="pl-c1">cd</span> <span class="pl-smi">${DATA_DIR}</span> <span class="pl-c"><span class="pl-c">#</span> download the Inception v3 model</span> curl -O http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz tar xzf inception-v3-2016-03-01.tar.gz <span class="pl-c"><span class="pl-c">#</span> this will create a directory called inception-v3 which contains the following files.</span> <span class="pl-k">></span> ls inception-v3 README.txt checkpoint model.ckpt-157585</pre></div> <p dir="auto"><a href="https://www.youtube.com/watch?v=9bZkp7q19f0" rel="nofollow">Congratulations!</a> You are now ready to fine-tune your pre-trained Inception v3 model with the flower data set.</p> <div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">How to Retrain a Trained Model on the Flowers Data</h3><a id="user-content-how-to-retrain-a-trained-model-on-the-flowers-data" class="anchor" aria-label="Permalink: How to Retrain a Trained Model on the Flowers Data" href="#how-to-retrain-a-trained-model-on-the-flowers-data"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">We are now ready to fine-tune a pre-trained Inception-v3 model on the flowers data set. This requires two distinct changes to our training procedure:</p> <ol dir="auto"> <li> <p dir="auto">Build the exact same model as previously except we change the number of labels in the final classification layer.</p> </li> <li> <p dir="auto">Restore all weights from the pre-trained Inception-v3 except for the final classification layer; this will get randomly initialized instead.</p> </li> </ol> <p dir="auto">We can perform these two operations by specifying two flags: <code>--pretrained_model_checkpoint_path</code> and <code>--fine_tune</code>. The first flag is a string that points to the path of a pre-trained Inception-v3 model. If this flag is specified, it will load the entire model from the checkpoint before the script begins training.</p> <p dir="auto">The second flag <code>--fine_tune</code> is a boolean that indicates whether the last classification layer should be randomly initialized or restored. You may set this flag to false if you wish to continue training a pre-trained model from a checkpoint. If you set this flag to true, you can train a new classification layer from scratch.</p> <p dir="auto">In order to understand how <code>--fine_tune</code> works, please see the discussion on <code>Variables</code> in the TensorFlow-Slim <a href="/wenwei202/terngrad/blob/master/inception/slim/README.md"><code>README.md</code></a>.</p> <p dir="auto">Putting this all together you can retrain a pre-trained Inception-v3 model on the flowers data set with the following command.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/flowers_train # Path to the downloaded Inception-v3 model. MODEL_PATH="${INCEPTION_MODEL_DIR}/model.ckpt-157585" # Directory where the flowers data resides. FLOWERS_DATA_DIR=/tmp/flowers-data/ # Directory where to save the checkpoint and events files. TRAIN_DIR=/tmp/flowers_train/ # Run the fine-tuning on the flowers data set starting from the pre-trained # Imagenet-v3 model. bazel-bin/inception/flowers_train \ --train_dir="${TRAIN_DIR}" \ --data_dir="${FLOWERS_DATA_DIR}" \ --pretrained_model_checkpoint_path="${MODEL_PATH}" \ --fine_tune=True \ --initial_learning_rate=0.001 \ --input_queue_memory_factor=1"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/flowers_train <span class="pl-c"><span class="pl-c">#</span> Path to the downloaded Inception-v3 model.</span> MODEL_PATH=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${INCEPTION_MODEL_DIR}</span>/model.ckpt-157585<span class="pl-pds">"</span></span> <span class="pl-c"><span class="pl-c">#</span> Directory where the flowers data resides.</span> FLOWERS_DATA_DIR=/tmp/flowers-data/ <span class="pl-c"><span class="pl-c">#</span> Directory where to save the checkpoint and events files.</span> TRAIN_DIR=/tmp/flowers_train/ <span class="pl-c"><span class="pl-c">#</span> Run the fine-tuning on the flowers data set starting from the pre-trained</span> <span class="pl-c"><span class="pl-c">#</span> Imagenet-v3 model.</span> bazel-bin/inception/flowers_train \ --train_dir=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${TRAIN_DIR}</span><span class="pl-pds">"</span></span> \ --data_dir=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${FLOWERS_DATA_DIR}</span><span class="pl-pds">"</span></span> \ --pretrained_model_checkpoint_path=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${MODEL_PATH}</span><span class="pl-pds">"</span></span> \ --fine_tune=True \ --initial_learning_rate=0.001 \ --input_queue_memory_factor=1</pre></div> <p dir="auto">We have added a few extra options to the training procedure.</p> <ul dir="auto"> <li>Fine-tuning a model a separate data set requires significantly lowering the initial learning rate. We set the initial learning rate to 0.001.</li> <li>The flowers data set is quite small so we shrink the size of the shuffling queue of examples. See <a href="#adjusting-memory-demands">Adjusting Memory Demands</a> for more details.</li> </ul> <p dir="auto">The training script will only reports the loss. To evaluate the quality of the fine-tuned model, you will need to run <code>flowers_eval</code>:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# Build the model. Note that we need to make sure the TensorFlow is ready to # use before this as this command will not build TensorFlow. bazel build inception/flowers_eval # Directory where we saved the fine-tuned checkpoint and events files. TRAIN_DIR=/tmp/flowers_train/ # Directory where the flowers data resides. FLOWERS_DATA_DIR=/tmp/flowers-data/ # Directory where to save the evaluation events files. EVAL_DIR=/tmp/flowers_eval/ # Evaluate the fine-tuned model on a hold-out of the flower data set. bazel-bin/inception/flowers_eval \ --eval_dir="${EVAL_DIR}" \ --data_dir="${FLOWERS_DATA_DIR}" \ --subset=validation \ --num_examples=500 \ --checkpoint_dir="${TRAIN_DIR}" \ --input_queue_memory_factor=1 \ --run_once"><pre><span class="pl-c"><span class="pl-c">#</span> Build the model. Note that we need to make sure the TensorFlow is ready to</span> <span class="pl-c"><span class="pl-c">#</span> use before this as this command will not build TensorFlow.</span> bazel build inception/flowers_eval <span class="pl-c"><span class="pl-c">#</span> Directory where we saved the fine-tuned checkpoint and events files.</span> TRAIN_DIR=/tmp/flowers_train/ <span class="pl-c"><span class="pl-c">#</span> Directory where the flowers data resides.</span> FLOWERS_DATA_DIR=/tmp/flowers-data/ <span class="pl-c"><span class="pl-c">#</span> Directory where to save the evaluation events files.</span> EVAL_DIR=/tmp/flowers_eval/ <span class="pl-c"><span class="pl-c">#</span> Evaluate the fine-tuned model on a hold-out of the flower data set.</span> bazel-bin/inception/flowers_eval \ --eval_dir=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${EVAL_DIR}</span><span class="pl-pds">"</span></span> \ --data_dir=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${FLOWERS_DATA_DIR}</span><span class="pl-pds">"</span></span> \ --subset=validation \ --num_examples=500 \ --checkpoint_dir=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${TRAIN_DIR}</span><span class="pl-pds">"</span></span> \ --input_queue_memory_factor=1 \ --run_once</pre></div> <p dir="auto">We find that the evaluation arrives at roughly 93.4% precision@1 after the model has been running for 2000 steps.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="Succesfully loaded model from /tmp/flowers/model.ckpt-1999 at step=1999. 2016-03-01 16:52:51.761219: starting evaluation on (validation). 2016-03-01 16:53:05.450419: [20 batches out of 20] (36.5 examples/sec; 0.684sec/batch) 2016-03-01 16:53:05.450471: precision @ 1 = 0.9340 recall @ 5 = 0.9960 [500 examples]"><pre>Succesfully loaded model from /tmp/flowers/model.ckpt-1999 at step=1999. 2016-03-01 16:52:51.761219: starting evaluation on (validation). 2016-03-01 16:53:05.450419: [20 batches out of 20] (36.5 examples/sec<span class="pl-k">;</span> 0.684sec/batch) 2016-03-01 16:53:05.450471: precision @ 1 = 0.9340 recall @ 5 = 0.9960 [500 examples]</pre></div> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">How to Construct a New Dataset for Retraining</h2><a id="user-content-how-to-construct-a-new-dataset-for-retraining" class="anchor" aria-label="Permalink: How to Construct a New Dataset for Retraining" href="#how-to-construct-a-new-dataset-for-retraining"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">One can use the existing scripts supplied with this model to build a new dataset for training or fine-tuning. The main script to employ is <a href="/wenwei202/terngrad/blob/master/inception/data/build_image_data.py"><code>build_image_data.py</code></a>. Briefly, this script takes a structured directory of images and converts it to a sharded <code>TFRecord</code> that can be read by the Inception model.</p> <p dir="auto">In particular, you will need to create a directory of training images that reside within <code>$TRAIN_DIR</code> and <code>$VALIDATION_DIR</code> arranged as such:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=" $TRAIN_DIR/dog/image0.jpeg $TRAIN_DIR/dog/image1.jpg $TRAIN_DIR/dog/image2.png ... $TRAIN_DIR/cat/weird-image.jpeg $TRAIN_DIR/cat/my-image.jpeg $TRAIN_DIR/cat/my-image.JPG ... $VALIDATION_DIR/dog/imageA.jpeg $VALIDATION_DIR/dog/imageB.jpg $VALIDATION_DIR/dog/imageC.png ... $VALIDATION_DIR/cat/weird-image.PNG $VALIDATION_DIR/cat/that-image.jpg $VALIDATION_DIR/cat/cat.JPG ..."><pre> <span class="pl-smi">$TRAIN_DIR</span>/dog/image0.jpeg <span class="pl-smi">$TRAIN_DIR</span>/dog/image1.jpg <span class="pl-smi">$TRAIN_DIR</span>/dog/image2.png ... <span class="pl-smi">$TRAIN_DIR</span>/cat/weird-image.jpeg <span class="pl-smi">$TRAIN_DIR</span>/cat/my-image.jpeg <span class="pl-smi">$TRAIN_DIR</span>/cat/my-image.JPG ... <span class="pl-smi">$VALIDATION_DIR</span>/dog/imageA.jpeg <span class="pl-smi">$VALIDATION_DIR</span>/dog/imageB.jpg <span class="pl-smi">$VALIDATION_DIR</span>/dog/imageC.png ... <span class="pl-smi">$VALIDATION_DIR</span>/cat/weird-image.PNG <span class="pl-smi">$VALIDATION_DIR</span>/cat/that-image.jpg <span class="pl-smi">$VALIDATION_DIR</span>/cat/cat.JPG ...</pre></div> <p dir="auto">Each sub-directory in <code>$TRAIN_DIR</code> and <code>$VALIDATION_DIR</code> corresponds to a unique label for the images that reside within that sub-directory. The images may be JPEG or PNG images. We do not support other images types currently.</p> <p dir="auto">Once the data is arranged in this directory structure, we can run <code>build_image_data.py</code> on the data to generate the sharded <code>TFRecord</code> dataset. Each entry of the <code>TFRecord</code> is a serialized <code>tf.Example</code> protocol buffer. A complete list of information contained in the <code>tf.Example</code> is described in the comments of <code>build_image_data.py</code>.</p> <p dir="auto">To run <code>build_image_data.py</code>, you can run the following command line:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="# location to where to save the TFRecord data. OUTPUT_DIRECTORY=$HOME/my-custom-data/ # build the preprocessing script. bazel build inception/build_image_data # convert the data. bazel-bin/inception/build_image_data \ --train_directory="${TRAIN_DIR}" \ --validation_directory="${VALIDATION_DIR}" \ --output_directory="${OUTPUT_DIRECTORY}" \ --labels_file="${LABELS_FILE}" \ --train_shards=128 \ --validation_shards=24 \ --num_threads=8"><pre><span class="pl-c"><span class="pl-c">#</span> location to where to save the TFRecord data.</span> OUTPUT_DIRECTORY=<span class="pl-smi">$HOME</span>/my-custom-data/ <span class="pl-c"><span class="pl-c">#</span> build the preprocessing script.</span> bazel build inception/build_image_data <span class="pl-c"><span class="pl-c">#</span> convert the data.</span> bazel-bin/inception/build_image_data \ --train_directory=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${TRAIN_DIR}</span><span class="pl-pds">"</span></span> \ --validation_directory=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${VALIDATION_DIR}</span><span class="pl-pds">"</span></span> \ --output_directory=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${OUTPUT_DIRECTORY}</span><span class="pl-pds">"</span></span> \ --labels_file=<span class="pl-s"><span class="pl-pds">"</span><span class="pl-smi">${LABELS_FILE}</span><span class="pl-pds">"</span></span> \ --train_shards=128 \ --validation_shards=24 \ --num_threads=8</pre></div> <p dir="auto">where the <code>$OUTPUT_DIRECTORY</code> is the location of the sharded <code>TFRecords</code>. The <code>$LABELS_FILE</code> will be a text file that is read by the script that provides a list of all of the labels. For instance, in the case flowers data set, the <code>$LABELS_FILE</code> contained the following data:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="daisy dandelion roses sunflowers tulips"><pre>daisy dandelion roses sunflowers tulips</pre></div> <p dir="auto">Note that each row of each label corresponds with the entry in the final classifier in the model. That is, the <code>daisy</code> corresponds to the classifier for entry <code>1</code>; <code>dandelion</code> is entry <code>2</code>, etc. We skip label <code>0</code> as a background class.</p> <p dir="auto">After running this script produces files that look like the following:</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=" $TRAIN_DIR/train-00000-of-00024 $TRAIN_DIR/train-00001-of-00024 ... $TRAIN_DIR/train-00023-of-00024 and $VALIDATION_DIR/validation-00000-of-00008 $VALIDATION_DIR/validation-00001-of-00008 ... $VALIDATION_DIR/validation-00007-of-00008"><pre> <span class="pl-smi">$TRAIN_DIR</span>/train-00000-of-00024 <span class="pl-smi">$TRAIN_DIR</span>/train-00001-of-00024 ... <span class="pl-smi">$TRAIN_DIR</span>/train-00023-of-00024 and <span class="pl-smi">$VALIDATION_DIR</span>/validation-00000-of-00008 <span class="pl-smi">$VALIDATION_DIR</span>/validation-00001-of-00008 ... <span class="pl-smi">$VALIDATION_DIR</span>/validation-00007-of-00008</pre></div> <p dir="auto">where 24 and 8 are the number of shards specified for each dataset, respectively. Generally speaking, we aim for selecting the number of shards such that roughly 1024 images reside in each shard. Once this data set is built, you are ready to train or fine-tune an Inception model on this data set.</p> <p dir="auto">Note, if you are piggy backing on the flowers retraining scripts, be sure to update <code>num_classes()</code> and <code>num_examples_per_epoch()</code> in <code>flowers_data.py</code> to correspond with your data.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Practical Considerations for Training a Model</h2><a id="user-content-practical-considerations-for-training-a-model" class="anchor" aria-label="Permalink: Practical Considerations for Training a Model" href="#practical-considerations-for-training-a-model"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">The model architecture and training procedure is heavily dependent on the hardware used to train the model. If you wish to train or fine-tune this model on your machine <strong>you will need to adjust and empirically determine a good set of training hyper-parameters for your setup</strong>. What follows are some general considerations for novices.</p> <div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Finding Good Hyperparameters</h3><a id="user-content-finding-good-hyperparameters" class="anchor" aria-label="Permalink: Finding Good Hyperparameters" href="#finding-good-hyperparameters"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Roughly 5-10 hyper-parameters govern the speed at which a network is trained. In addition to <code>--batch_size</code> and <code>--num_gpus</code>, there are several constants defined in <a href="/wenwei202/terngrad/blob/master/inception/inception_train.py">inception_train.py</a> which dictate the learning schedule.</p> <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="RMSPROP_DECAY = 0.9 # Decay term for RMSProp. MOMENTUM = 0.9 # Momentum in RMSProp. RMSPROP_EPSILON = 1.0 # Epsilon term for RMSProp. INITIAL_LEARNING_RATE = 0.1 # Initial learning rate. NUM_EPOCHS_PER_DECAY = 30.0 # Epochs after which learning rate decays. LEARNING_RATE_DECAY_FACTOR = 0.16 # Learning rate decay factor."><pre>RMSPROP_DECAY = 0.9 <span class="pl-c"><span class="pl-c">#</span> Decay term for RMSProp.</span> MOMENTUM = 0.9 <span class="pl-c"><span class="pl-c">#</span> Momentum in RMSProp.</span> RMSPROP_EPSILON = 1.0 <span class="pl-c"><span class="pl-c">#</span> Epsilon term for RMSProp.</span> INITIAL_LEARNING_RATE = 0.1 <span class="pl-c"><span class="pl-c">#</span> Initial learning rate.</span> NUM_EPOCHS_PER_DECAY = 30.0 <span class="pl-c"><span class="pl-c">#</span> Epochs after which learning rate decays.</span> LEARNING_RATE_DECAY_FACTOR = 0.16 <span class="pl-c"><span class="pl-c">#</span> Learning rate decay factor.</span></pre></div> <p dir="auto">There are many papers that discuss the various tricks and trade-offs associated with training a model with stochastic gradient descent. For those new to the field, some great references are:</p> <ul dir="auto"> <li>Y Bengio, <a href="http://arxiv.org/abs/1206.5533" rel="nofollow">Practical recommendations for gradient-based training of deep architectures</a></li> <li>I Goodfellow, Y Bengio and A Courville, [Deep Learning] (<a href="http://www.deeplearningbook.org/" rel="nofollow">http://www.deeplearningbook.org/</a>)</li> </ul> <p dir="auto">What follows is a summary of some general advice for identifying appropriate model hyper-parameters in the context of this particular model training setup. Namely, this library provides <em>synchronous</em> updates to model parameters based on batch-splitting the model across multiple GPUs.</p> <ul dir="auto"> <li> <p dir="auto">Higher learning rates leads to faster training. Too high of learning rate leads to instability and will cause model parameters to diverge to infinity or NaN.</p> </li> <li> <p dir="auto">Larger batch sizes lead to higher quality estimates of the gradient and permit training the model with higher learning rates.</p> </li> <li> <p dir="auto">Often the GPU memory is a bottleneck that prevents employing larger batch sizes. Employing more GPUs allows one to user larger batch sizes because this model splits the batch across the GPUs.</p> </li> </ul> <p dir="auto"><strong>NOTE</strong> If one wishes to train this model with <em>asynchronous</em> gradient updates, one will need to substantially alter this model and new considerations need to be factored into hyperparameter tuning. See <a href="http://research.google.com/archive/large_deep_networks_nips2012.html" rel="nofollow">Large Scale Distributed Deep Networks</a> for a discussion in this domain.</p> <div class="markdown-heading" dir="auto"><h3 tabindex="-1" class="heading-element" dir="auto">Adjusting Memory Demands</h3><a id="user-content-adjusting-memory-demands" class="anchor" aria-label="Permalink: Adjusting Memory Demands" href="#adjusting-memory-demands"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">Training this model has large memory demands in terms of the CPU and GPU. Let's discuss each item in turn.</p> <p dir="auto">GPU memory is relatively small compared to CPU memory. Two items dictate the amount of GPU memory employed -- model architecture and batch size. Assuming that you keep the model architecture fixed, the sole parameter governing the GPU demand is the batch size. A good rule of thumb is to try employ as large of batch size as will fit on the GPU.</p> <p dir="auto">If you run out of GPU memory, either lower the <code>--batch_size</code> or employ more GPUs on your desktop. The model performs batch-splitting across GPUs, thus N GPUs can handle N times the batch size of 1 GPU.</p> <p dir="auto">The model requires a large amount of CPU memory as well. We have tuned the model to employ about ~20GB of CPU memory. Thus, having access to about 40 GB of CPU memory would be ideal.</p> <p dir="auto">If that is not possible, you can tune down the memory demands of the model via lowering <code>--input_queue_memory_factor</code>. Images are preprocessed asynchronously with respect to the main training across <code>--num_preprocess_threads</code> threads. The preprocessed images are stored in shuffling queue in which each GPU performs a dequeue operation in order to receive a <code>batch_size</code> worth of images.</p> <p dir="auto">In order to guarantee good shuffling across the data, we maintain a large shuffling queue of 1024 x <code>input_queue_memory_factor</code> images. For the current model architecture, this corresponds to about 4GB of CPU memory. You may lower <code>input_queue_memory_factor</code> in order to decrease the memory footprint. Keep in mind though that lowering this value drastically may result in a model with slightly lower predictive accuracy when training from scratch. Please see comments in <a href="/wenwei202/terngrad/blob/master/inception/image_processing.py"><code>image_processing.py</code></a> for more details.</p> <div class="markdown-heading" dir="auto"><h2 tabindex="-1" class="heading-element" dir="auto">Troubleshooting</h2><a id="user-content-troubleshooting" class="anchor" aria-label="Permalink: Troubleshooting" href="#troubleshooting"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">The model runs out of CPU memory.</h4><a id="user-content-the-model-runs-out-of-cpu-memory" class="anchor" aria-label="Permalink: The model runs out of CPU memory." href="#the-model-runs-out-of-cpu-memory"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">In lieu of buying more CPU memory, an easy fix is to decrease <code>--input_queue_memory_factor</code>. See [Adjusting Memory Demands] (#adjusting-memory-demands).</p> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">The model runs out of GPU memory.</h4><a id="user-content-the-model-runs-out-of-gpu-memory" class="anchor" aria-label="Permalink: The model runs out of GPU memory." href="#the-model-runs-out-of-gpu-memory"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">The data is not able to fit on the GPU card. The simplest solution is to decrease the batch size of the model. Otherwise, you will need to think about a more sophisticated method for specifying the training which cuts up the model across multiple <code>session.run()</code> calls or partitions the model across multiple GPUs. See <a href="https://www.tensorflow.org/how_tos/using_gpu/index.html" rel="nofollow">Using GPUs</a> and <a href="#adjusting-memory-demands">Adjusting Memory Demands</a> for more information.</p> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">The model training results in NaN's.</h4><a id="user-content-the-model-training-results-in-nans" class="anchor" aria-label="Permalink: The model training results in NaN's." href="#the-model-training-results-in-nans"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">The learning rate of the model is too high. Turn down your learning rate.</p> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">I wish to train a model with a different image size.</h4><a id="user-content-i-wish-to-train-a-model-with-a-different-image-size" class="anchor" aria-label="Permalink: I wish to train a model with a different image size." href="#i-wish-to-train-a-model-with-a-different-image-size"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">The simplest solution is to artificially resize your images to <code>299x299</code> pixels. See <a href="https://www.tensorflow.org/api_docs/python/image.html" rel="nofollow">Images</a> section for many resizing, cropping and padding methods. Note that the entire model architecture is predicated on a <code>299x299</code> image, thus if you wish to change the input image size, then you may need to redesign the entire model architecture.</p> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">What hardware specification are these hyper-parameters targeted for?</h4><a id="user-content-what-hardware-specification-are-these-hyper-parameters-targeted-for" class="anchor" aria-label="Permalink: What hardware specification are these hyper-parameters targeted for?" href="#what-hardware-specification-are-these-hyper-parameters-targeted-for"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">We targeted a desktop with 128GB of CPU ram connected to 8 NVIDIA Tesla K40 GPU cards but we have run this on desktops with 32GB of CPU ram and 1 NVIDIA Tesla K40. You can get a sense of the various training configurations we tested by reading the comments in <a href="/wenwei202/terngrad/blob/master/inception/inception_train.py"><code>inception_train.py</code></a>.</p> <div class="markdown-heading" dir="auto"><h4 tabindex="-1" class="heading-element" dir="auto">How do I continue training from a checkpoint in distributed setting?</h4><a id="user-content-how-do-i-continue-training-from-a-checkpoint-in-distributed-setting" class="anchor" aria-label="Permalink: How do I continue training from a checkpoint in distributed setting?" href="#how-do-i-continue-training-from-a-checkpoint-in-distributed-setting"><svg class="octicon octicon-link" viewBox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div> <p dir="auto">You only need to make sure that the checkpoint is in a location that can be reached by all of the <code>ps</code> tasks. By specifying the checkpoint location with <code>--train_dir</code> , the <code>ps</code> servers will load the checkpoint before commencing training.</p> </article></div></div></div></div></div> <!-- --> <!-- --> <script type="application/json" id="__PRIMER_DATA_:R0:__">{"resolvedServerColorMode":"day"}</script></div> </react-partial> <input type="hidden" data-csrf="true" value="K/A4Gobve2E54A31+PiIf7OkLXFSwTtPvBFqIdQc+AI4sPmgq+8u+0LG4NJeXrdAV11dP8dja5ZxYg3uLJhLeQ==" /> </div> <div data-view-component="true" class="Layout-sidebar"> <div class="BorderGrid about-margin" data-pjax> <div class="BorderGrid-row"> <div class="BorderGrid-cell"> <div class="hide-sm hide-md"> <h2 class="mb-3 h4">About</h2> <p class="f4 my-3"> Ternary Gradients to Reduce Communication in Distributed Deep Learning (TensorFlow) </p> <h3 class="sr-only">Topics</h3> <div class="my-3"> <div class="f6"> <a href="/topics/deep-neural-networks" title="Topic: deep-neural-networks" data-view-component="true" class="topic-tag topic-tag-link"> deep-neural-networks </a> <a href="/topics/deep-learning" title="Topic: deep-learning" data-view-component="true" class="topic-tag topic-tag-link"> deep-learning </a> <a href="/topics/sgd" title="Topic: sgd" data-view-component="true" class="topic-tag topic-tag-link"> sgd </a> <a href="/topics/quantization" title="Topic: quantization" data-view-component="true" class="topic-tag topic-tag-link"> quantization </a> <a href="/topics/data-parallelism" title="Topic: data-parallelism" data-view-component="true" class="topic-tag topic-tag-link"> data-parallelism </a> <a href="/topics/distributed-training" title="Topic: distributed-training" data-view-component="true" class="topic-tag topic-tag-link"> distributed-training </a> </div> </div> <h3 class="sr-only">Resources</h3> <div class="mt-2"> <a class="Link--muted" data-analytics-event="{"category":"Repository Overview","action":"click","label":"location:sidebar;file:readme"}" href="#readme-ov-file"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-book mr-2"> <path d="M0 1.75A.75.75 0 0 1 .75 1h4.253c1.227 0 2.317.59 3 1.501A3.743 3.743 0 0 1 11.006 1h4.245a.75.75 0 0 1 .75.75v10.5a.75.75 0 0 1-.75.75h-4.507a2.25 2.25 0 0 0-1.591.659l-.622.621a.75.75 0 0 1-1.06 0l-.622-.621A2.25 2.25 0 0 0 5.258 13H.75a.75.75 0 0 1-.75-.75Zm7.251 10.324.004-5.073-.002-2.253A2.25 2.25 0 0 0 5.003 2.5H1.5v9h3.757a3.75 3.75 0 0 1 1.994.574ZM8.755 4.75l-.004 7.322a3.752 3.752 0 0 1 1.992-.572H14.5v-9h-3.495a2.25 2.25 0 0 0-2.25 2.25Z"></path> </svg> Readme </a> </div> <h3 class="sr-only">License</h3> <div class="mt-2"> <a href="#Apache-2.0-1-ov-file" class="Link--muted" data-analytics-event="{"category":"Repository Overview","action":"click","label":"location:sidebar;file:license"}" > <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-law mr-2"> <path d="M8.75.75V2h.985c.304 0 .603.08.867.231l1.29.736c.038.022.08.033.124.033h2.234a.75.75 0 0 1 0 1.5h-.427l2.111 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.006.005-.01.01-.045.04c-.21.176-.441.327-.686.45C14.556 10.78 13.88 11 13 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L12.178 4.5h-.162c-.305 0-.604-.079-.868-.231l-1.29-.736a.245.245 0 0 0-.124-.033H8.75V13h2.5a.75.75 0 0 1 0 1.5h-6.5a.75.75 0 0 1 0-1.5h2.5V3.5h-.984a.245.245 0 0 0-.124.033l-1.289.737c-.265.15-.564.23-.869.23h-.162l2.112 4.692a.75.75 0 0 1-.154.838l-.53-.53.529.531-.001.002-.002.002-.006.006-.016.015-.045.04c-.21.176-.441.327-.686.45C4.556 10.78 3.88 11 3 11a4.498 4.498 0 0 1-2.023-.454 3.544 3.544 0 0 1-.686-.45l-.045-.04-.016-.015-.006-.006-.004-.004v-.001a.75.75 0 0 1-.154-.838L2.178 4.5H1.75a.75.75 0 0 1 0-1.5h2.234a.249.249 0 0 0 .125-.033l1.288-.737c.265-.15.564-.23.869-.23h.984V.75a.75.75 0 0 1 1.5 0Zm2.945 8.477c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L13 6.327Zm-10 0c.285.135.718.273 1.305.273s1.02-.138 1.305-.273L3 6.327Z"></path> </svg> Apache-2.0 license </a> </div> <include-fragment src="/wenwei202/terngrad/hovercards/citation/sidebar_partial?tree_name=master"> </include-fragment> <div class="mt-2"> <a href="/wenwei202/terngrad/activity" data-view-component="true" class="Link Link--muted"><svg text="gray" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-pulse mr-2"> <path d="M6 2c.306 0 .582.187.696.471L10 10.731l1.304-3.26A.751.751 0 0 1 12 7h3.25a.75.75 0 0 1 0 1.5h-2.742l-1.812 4.528a.751.751 0 0 1-1.392 0L6 4.77 4.696 8.03A.75.75 0 0 1 4 8.5H.75a.75.75 0 0 1 0-1.5h2.742l1.812-4.529A.751.751 0 0 1 6 2Z"></path> </svg> <span class="color-fg-muted">Activity</span></a> </div> <h3 class="sr-only">Stars</h3> <div class="mt-2"> <a href="/wenwei202/terngrad/stargazers" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-star mr-2"> <path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path> </svg> <strong>182</strong> stars</a> </div> <h3 class="sr-only">Watchers</h3> <div class="mt-2"> <a href="/wenwei202/terngrad/watchers" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-eye mr-2"> <path d="M8 2c1.981 0 3.671.992 4.933 2.078 1.27 1.091 2.187 2.345 2.637 3.023a1.62 1.62 0 0 1 0 1.798c-.45.678-1.367 1.932-2.637 3.023C11.67 13.008 9.981 14 8 14c-1.981 0-3.671-.992-4.933-2.078C1.797 10.83.88 9.576.43 8.898a1.62 1.62 0 0 1 0-1.798c.45-.677 1.367-1.931 2.637-3.022C4.33 2.992 6.019 2 8 2ZM1.679 7.932a.12.12 0 0 0 0 .136c.411.622 1.241 1.75 2.366 2.717C5.176 11.758 6.527 12.5 8 12.5c1.473 0 2.825-.742 3.955-1.715 1.124-.967 1.954-2.096 2.366-2.717a.12.12 0 0 0 0-.136c-.412-.621-1.242-1.75-2.366-2.717C10.824 4.242 9.473 3.5 8 3.5c-1.473 0-2.825.742-3.955 1.715-1.124.967-1.954 2.096-2.366 2.717ZM8 10a2 2 0 1 1-.001-3.999A2 2 0 0 1 8 10Z"></path> </svg> <strong>11</strong> watching</a> </div> <h3 class="sr-only">Forks</h3> <div class="mt-2"> <a href="/wenwei202/terngrad/forks" data-view-component="true" class="Link Link--muted"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-repo-forked mr-2"> <path d="M5 5.372v.878c0 .414.336.75.75.75h4.5a.75.75 0 0 0 .75-.75v-.878a2.25 2.25 0 1 1 1.5 0v.878a2.25 2.25 0 0 1-2.25 2.25h-1.5v2.128a2.251 2.251 0 1 1-1.5 0V8.5h-1.5A2.25 2.25 0 0 1 3.5 6.25v-.878a2.25 2.25 0 1 1 1.5 0ZM5 3.25a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Zm6.75.75a.75.75 0 1 0 0-1.5.75.75 0 0 0 0 1.5Zm-3 8.75a.75.75 0 1 0-1.5 0 .75.75 0 0 0 1.5 0Z"></path> </svg> <strong>48</strong> forks</a> </div> <div class="mt-2"> <a class="Link--muted" href="/contact/report-content?content_url=https%3A%2F%2Fgithub.com%2Fwenwei202%2Fterngrad&report=wenwei202+%28user%29"> Report repository </a> </div> </div> </div> </div> <div class="BorderGrid-row"> <div class="BorderGrid-cell"> <h2 class="h4 mb-3" data-pjax="#repo-content-pjax-container" data-turbo-frame="repo-content-turbo-frame"> <a href="/wenwei202/terngrad/releases" data-view-component="true" class="Link--primary no-underline Link">Releases</a></h2> <div class="text-small color-fg-muted">No releases published</div> </div> </div> <div class="BorderGrid-row"> <div class="BorderGrid-cell"> <h2 class="h4 mb-3"> <a href="/users/wenwei202/packages?repo_name=terngrad" data-view-component="true" class="Link--primary no-underline Link d-flex flex-items-center">Packages <span title="0" hidden="hidden" data-view-component="true" class="Counter ml-1">0</span></a></h2> <div class="text-small color-fg-muted" > No packages published <br> </div> </div> </div> <div class="BorderGrid-row" hidden> <div class="BorderGrid-cell"> <include-fragment src="/wenwei202/terngrad/used_by_list" accept="text/fragment+html"> </include-fragment> </div> </div> <div class="BorderGrid-row"> <div class="BorderGrid-cell"> <h2 class="h4 mb-3"> <a href="/wenwei202/terngrad/graphs/contributors" data-view-component="true" class="Link--primary no-underline Link d-flex flex-items-center">Contributors <span title="27" data-view-component="true" class="Counter ml-1">27</span></a></h2> <include-fragment src="/wenwei202/terngrad/contributors_list?count=27&current_repository=terngrad&items_to_show=14" aria-busy="true" aria-label="Loading contributors"> <ul class="list-style-none d-flex flex-wrap mb-n2"> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> <li class="mb-2 "> <div class="Skeleton avatar avatar-user mr-2" style="width:32px;height:32px;"></div> </li> </ul> </include-fragment> <div data-view-component="true" class="mt-3"> <a text="small" href="/wenwei202/terngrad/graphs/contributors" data-view-component="true" class="Link--inTextBlock Link">+ 13 contributors</a></div> </div> </div> <div class="BorderGrid-row"> <div class="BorderGrid-cell"> <h2 class="h4 mb-3">Languages</h2> <div class="mb-2"> <span data-view-component="true" class="Progress"> <span style="background-color:#3572A5 !important;;width: 88.4%;" itemprop="keywords" data-view-component="true" class="Progress-item color-bg-success-emphasis"></span> <span style="background-color:#89e051 !important;;width: 6.9%;" itemprop="keywords" data-view-component="true" class="Progress-item color-bg-success-emphasis"></span> <span style="background-color:#DA5B0B !important;;width: 4.7%;" itemprop="keywords" data-view-component="true" class="Progress-item color-bg-success-emphasis"></span> </span></div> <ul class="list-style-none"> <li class="d-inline"> <a class="d-inline-flex flex-items-center flex-nowrap Link--secondary no-underline text-small mr-3" href="/wenwei202/terngrad/search?l=python" data-ga-click="Repository, language stats search click, location:repo overview"> <svg style="color:#3572A5;" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill mr-2"> <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path> </svg> <span class="color-fg-default text-bold mr-1">Python</span> <span>88.4%</span> </a> </li> <li class="d-inline"> <a class="d-inline-flex flex-items-center flex-nowrap Link--secondary no-underline text-small mr-3" href="/wenwei202/terngrad/search?l=shell" data-ga-click="Repository, language stats search click, location:repo overview"> <svg style="color:#89e051;" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill mr-2"> <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path> </svg> <span class="color-fg-default text-bold mr-1">Shell</span> <span>6.9%</span> </a> </li> <li class="d-inline"> <a class="d-inline-flex flex-items-center flex-nowrap Link--secondary no-underline text-small mr-3" href="/wenwei202/terngrad/search?l=jupyter-notebook" data-ga-click="Repository, language stats search click, location:repo overview"> <svg style="color:#DA5B0B;" aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-dot-fill mr-2"> <path d="M8 4a4 4 0 1 1 0 8 4 4 0 0 1 0-8Z"></path> </svg> <span class="color-fg-default text-bold mr-1">Jupyter Notebook</span> <span>4.7%</span> </a> </li> </ul> </div> </div> </div> </div> </div></div> </div> </div> </turbo-frame> </main> </div> </div> <footer class="footer pt-8 pb-6 f6 color-fg-muted p-responsive" role="contentinfo" > <h2 class='sr-only'>Footer</h2> <div class="d-flex flex-justify-center flex-items-center flex-column-reverse flex-lg-row flex-wrap flex-lg-nowrap"> <div class="d-flex flex-items-center flex-shrink-0 mx-2"> <a aria-label="Homepage" title="GitHub" class="footer-octicon mr-2" href="https://github.com"> <svg aria-hidden="true" height="24" viewBox="0 0 24 24" version="1.1" width="24" data-view-component="true" class="octicon octicon-mark-github"> <path d="M12 1C5.9225 1 1 5.9225 1 12C1 16.8675 4.14875 20.9787 8.52125 22.4362C9.07125 22.5325 9.2775 22.2025 9.2775 21.9137C9.2775 21.6525 9.26375 20.7862 9.26375 19.865C6.5 20.3737 5.785 19.1912 5.565 18.5725C5.44125 18.2562 4.905 17.28 4.4375 17.0187C4.0525 16.8125 3.5025 16.3037 4.42375 16.29C5.29 16.2762 5.90875 17.0875 6.115 17.4175C7.105 19.0812 8.68625 18.6137 9.31875 18.325C9.415 17.61 9.70375 17.1287 10.02 16.8537C7.5725 16.5787 5.015 15.63 5.015 11.4225C5.015 10.2262 5.44125 9.23625 6.1425 8.46625C6.0325 8.19125 5.6475 7.06375 6.2525 5.55125C6.2525 5.55125 7.17375 5.2625 9.2775 6.67875C10.1575 6.43125 11.0925 6.3075 12.0275 6.3075C12.9625 6.3075 13.8975 6.43125 14.7775 6.67875C16.8813 5.24875 17.8025 5.55125 17.8025 5.55125C18.4075 7.06375 18.0225 8.19125 17.9125 8.46625C18.6138 9.23625 19.04 10.2125 19.04 11.4225C19.04 15.6437 16.4688 16.5787 14.0213 16.8537C14.42 17.1975 14.7638 17.8575 14.7638 18.8887C14.7638 20.36 14.75 21.5425 14.75 21.9137C14.75 22.2025 14.9563 22.5462 15.5063 22.4362C19.8513 20.9787 23 16.8537 23 12C23 5.9225 18.0775 1 12 1Z"></path> </svg> </a> <span> © 2025 GitHub, Inc. </span> </div> <nav aria-label="Footer"> <h3 class="sr-only" id="sr-footer-heading">Footer navigation</h3> <ul class="list-style-none d-flex flex-justify-center flex-wrap mb-2 mb-lg-0" aria-labelledby="sr-footer-heading"> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to Terms","label":"text:terms"}" href="https://docs.github.com/site-policy/github-terms/github-terms-of-service" data-view-component="true" class="Link--secondary Link">Terms</a> </li> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to privacy","label":"text:privacy"}" href="https://docs.github.com/site-policy/privacy-policies/github-privacy-statement" data-view-component="true" class="Link--secondary Link">Privacy</a> </li> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to security","label":"text:security"}" href="https://github.com/security" data-view-component="true" class="Link--secondary Link">Security</a> </li> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to status","label":"text:status"}" href="https://www.githubstatus.com/" data-view-component="true" class="Link--secondary Link">Status</a> </li> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to docs","label":"text:docs"}" href="https://docs.github.com/" data-view-component="true" class="Link--secondary Link">Docs</a> </li> <li class="mx-2"> <a data-analytics-event="{"category":"Footer","action":"go to contact","label":"text:contact"}" href="https://support.github.com?tags=dotcom-footer" data-view-component="true" class="Link--secondary Link">Contact</a> </li> <li class="mx-2" > <cookie-consent-link> <button type="button" class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent" data-action="click:cookie-consent-link#showConsentManagement" data-analytics-event="{"location":"footer","action":"cookies","context":"subfooter","tag":"link","label":"cookies_link_subfooter_footer"}" > Manage cookies </button> </cookie-consent-link> </li> <li class="mx-2"> <cookie-consent-link> <button type="button" class="Link--secondary underline-on-hover border-0 p-0 color-bg-transparent" data-action="click:cookie-consent-link#showConsentManagement" data-analytics-event="{"location":"footer","action":"dont_share_info","context":"subfooter","tag":"link","label":"dont_share_info_link_subfooter_footer"}" > Do not share my personal information </button> </cookie-consent-link> </li> </ul> </nav> </div> </footer> <ghcc-consent id="ghcc" class="position-fixed bottom-0 left-0" style="z-index: 999999" data-initial-cookie-consent-allowed="" data-cookie-consent-required="false"></ghcc-consent> <div id="ajax-error-message" class="ajax-error-message flash flash-error" hidden> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-alert"> <path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path> </svg> <button type="button" class="flash-close js-ajax-error-dismiss" aria-label="Dismiss error"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg> </button> You can’t perform that action at this time. </div> <template id="site-details-dialog"> <details class="details-reset details-overlay details-overlay-dark lh-default color-fg-default hx_rsm" open> <summary role="button" aria-label="Close dialog"></summary> <details-dialog class="Box Box--overlay d-flex flex-column anim-fade-in fast hx_rsm-dialog hx_rsm-modal"> <button class="Box-btn-octicon m-0 btn-octicon position-absolute right-0 top-0" type="button" aria-label="Close dialog" data-close-dialog> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x"> <path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path> </svg> </button> <div class="octocat-spinner my-6 js-details-dialog-spinner"></div> </details-dialog> </details> </template> <div class="Popover js-hovercard-content position-absolute" style="display: none; outline: none;"> <div class="Popover-message Popover-message--bottom-left Popover-message--large Box color-shadow-large" style="width:360px;"> </div> </div> <template id="snippet-clipboard-copy-button"> <div class="zeroclipboard-container position-absolute right-0 top-0"> <clipboard-copy aria-label="Copy" class="ClipboardButton btn js-clipboard-copy m-2 p-0" data-copy-feedback="Copied!" data-tooltip-direction="w"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon m-2"> <path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path> </svg> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none m-2"> <path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path> </svg> </clipboard-copy> </div> </template> <template id="snippet-clipboard-copy-button-unpositioned"> <div class="zeroclipboard-container"> <clipboard-copy aria-label="Copy" class="ClipboardButton btn btn-invisible js-clipboard-copy m-2 p-0 d-flex flex-justify-center flex-items-center" data-copy-feedback="Copied!" data-tooltip-direction="w"> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-copy js-clipboard-copy-icon"> <path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"></path><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"></path> </svg> <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-check js-clipboard-check-icon color-fg-success d-none"> <path d="M13.78 4.22a.75.75 0 0 1 0 1.06l-7.25 7.25a.75.75 0 0 1-1.06 0L2.22 9.28a.751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018L6 10.94l6.72-6.72a.75.75 0 0 1 1.06 0Z"></path> </svg> </clipboard-copy> </div> </template> </div> <div id="js-global-screen-reader-notice" class="sr-only mt-n1" aria-live="polite" aria-atomic="true" ></div> <div id="js-global-screen-reader-notice-assertive" class="sr-only mt-n1" aria-live="assertive" aria-atomic="true"></div> </body> </html>