CINXE.COM

<!DOCTYPE html>     <html lang="en-US">  <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width" /> <title> July | 2014 | code.flickr.com </title> <link rel="profile" href="https://gmpg.org/xfn/11" /> <link rel="stylesheet" type="text/css" media="all" href="https://code.flickr.net/wp-content/themes/flickr-code/style.css?ver=20240716" /> <link rel="pingback" href="https://code.flickr.net/xmlrpc.php">  <meta name='robots' content='max-image-preview:large' /> <style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style> <link rel='dns-prefetch' href='//stats.wp.com' /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Feed" href="https://code.flickr.net/feed/" /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Comments Feed" href="https://code.flickr.net/comments/feed/" /> <script type="text/javascript"> /* <![CDATA[ */ window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/code.flickr.net\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.7.2"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83d\udc26\u200d\u2b1b","\ud83d\udc26\u200b\u2b1b")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); /* ]]> */ </script> <style id='wp-emoji-styles-inline-css'> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='all-css-2' href='https://code.flickr.net/wp-includes/css/dist/block-library/style.min.css?m=1742994400g' type='text/css' media='all' /> <style id='wp-block-library-theme-inline-css'> .wp-block-audio :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-audio :where(figcaption){color:#ffffffa6}.wp-block-audio{margin:0 0 1em}.wp-block-code{border:1px solid #ccc;border-radius:4px;font-family:Menlo,Consolas,monaco,monospace;padding:.8em 1em}.wp-block-embed :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-embed :where(figcaption){color:#ffffffa6}.wp-block-embed{margin:0 0 1em}.blocks-gallery-caption{color:#555;font-size:13px;text-align:center}.is-dark-theme .blocks-gallery-caption{color:#ffffffa6}:root :where(.wp-block-image figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme :root :where(.wp-block-image figcaption){color:#ffffffa6}.wp-block-image{margin:0 0 1em}.wp-block-pullquote{border-bottom:4px solid;border-top:4px solid;color:currentColor;margin-bottom:1.75em}.wp-block-pullquote cite,.wp-block-pullquote footer,.wp-block-pullquote__citation{color:currentColor;font-size:.8125em;font-style:normal;text-transform:uppercase}.wp-block-quote{border-left:.25em solid;margin:0 0 1.75em;padding-left:1em}.wp-block-quote cite,.wp-block-quote footer{color:currentColor;font-size:.8125em;font-style:normal;position:relative}.wp-block-quote:where(.has-text-align-right){border-left:none;border-right:.25em solid;padding-left:0;padding-right:1em}.wp-block-quote:where(.has-text-align-center){border:none;padding-left:0}.wp-block-quote.is-large,.wp-block-quote.is-style-large,.wp-block-quote:where(.is-style-plain){border:none}.wp-block-search .wp-block-search__label{font-weight:700}.wp-block-search__button{border:1px solid #ccc;padding:.375em .625em}:where(.wp-block-group.has-background){padding:1.25em 2.375em}.wp-block-separator.has-css-opacity{opacity:.4}.wp-block-separator{border:none;border-bottom:2px solid;margin-left:auto;margin-right:auto}.wp-block-separator.has-alpha-channel-opacity{opacity:1}.wp-block-separator:not(.is-style-wide):not(.is-style-dots){width:100px}.wp-block-separator.has-background:not(.is-style-dots){border-bottom:none;height:1px}.wp-block-separator.has-background:not(.is-style-wide):not(.is-style-dots){height:2px}.wp-block-table{margin:0 0 1em}.wp-block-table td,.wp-block-table th{word-break:normal}.wp-block-table :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-table :where(figcaption){color:#ffffffa6}.wp-block-video :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-video :where(figcaption){color:#ffffffa6}.wp-block-video{margin:0 0 1em}:root :where(.wp-block-template-part.has-background){margin-bottom:0;margin-top:0;padding:1.25em 2.375em} </style> <link rel='stylesheet' id='all-css-6' href='https://code.flickr.net/_static/??-eJzTLy/QzcxLzilNSS3WzyrWz01NyUxMzUnNTc0rQeEU5CRWphbp5qSmJyZX6uVm5uklFxfr6OPTDpRD5sM02efaGpqbGFlampgYGAIAROAu5A==' type='text/css' media='all' /> <style id='jetpack-sharing-buttons-style-inline-css'> .jetpack-sharing-buttons__services-list{display:flex;flex-direction:row;flex-wrap:wrap;gap:0;list-style-type:none;margin:5px;padding:0}.jetpack-sharing-buttons__services-list.has-small-icon-size{font-size:12px}.jetpack-sharing-buttons__services-list.has-normal-icon-size{font-size:16px}.jetpack-sharing-buttons__services-list.has-large-icon-size{font-size:24px}.jetpack-sharing-buttons__services-list.has-huge-icon-size{font-size:36px}@media print{.jetpack-sharing-buttons__services-list{display:none!important}}.editor-styles-wrapper .wp-block-jetpack-sharing-buttons{gap:0;padding-inline-start:0}ul.jetpack-sharing-buttons__services-list.has-background{padding:1.25em 2.375em} </style> <style id='classic-theme-styles-inline-css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css'> :root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--blue: #1982d1;--wp--preset--color--dark-gray: #373737;--wp--preset--color--medium-gray: #666;--wp--preset--color--light-gray: #e2e2e2;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} :root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;} </style> <link rel='stylesheet' id='all-css-12' href='https://code.flickr.net/_static/??-eJzTLy/QTc7PK0nNK9EvyUjNTS3WLykHcipTc1LLUvP0i0sqc1L1kouLdfQxVablZCZnFwFFU1LxK0QxMiknPzm7GKTUPtfW0NzEwMTQ2MLSEABabDMX' type='text/css' media='all' /> <link rel="https://api.w.org/" href="https://code.flickr.net/wp-json/" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://code.flickr.net/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.7.2" /> <style>img#wpstats{display:none}</style> <style type="text/css" id="twentyeleven-header-css"> #site-title, #site-description { position: absolute; clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ clip: rect(1px, 1px, 1px, 1px); } </style> <style type="text/css" id="wp-custom-css"> body { background: #fff; } body,input,textarea { font-family: Arial, sans-serif; color: #404040; } #page { margin: 1em auto; max-width: 1000px; } a:link { text-decoration: none; color: #0063dc; } a:visited { text-decoration: none; color: #0063dc; } a:hover { text-decoration: none; background: #0063dc; color: #fff; } a:active { text-decoration: none; background-color: #0259c4; color: #fff; } #branding { border: none; } #branding .only-search #s,#branding .only-search #s:focus { width: 280px; background-color: transparent; border-color: #ddd; } #branding #searchform { right: 0; } nav#access { background: none; box-shadow: none; -webkit-box-shadow: none; -moz-box-shadow: none; margin: 0; } nav#access a { font-weight: bold; margin: 11px 28px 0 0; padding: 0; line-height: 21px; } nav#access a:link,nav#access a:visited { text-decoration: none; color: #888; } nav#access a:hover,nav#access li:hover > a,nav#access a:focus,nav#access a:active { text-decoration: none; color: #0063dc; background: none; } #branding .only-search + nav#access div { padding-right: 300px; } nav#access div,nav#access ul { margin: 0; } @media screen and (max-width: 768px) { #branding .only-search + nav#access div { padding-right: 0; } #branding #searchform { display: none; position: static; text-align: center; } #branding .with-image #searchform { max-width: 100%; } #branding .only-search #s,#branding .only-search #s:focus { width: 85%; float: none; } nav#access { margin: 0 0 0 5%; } } @media screen and (max-width: 480px) { nav#access { display: none; } } #content { margin: 0 20% 0 0; width: 80%; } #content .comments-link { display: none; } @media screen and (max-width: 800px) { #main { padding-top: 0; overflow: hidden; } #branding { padding-bottom: 0; } #main #content { margin: 0; } .entry-title,.entry-header .entry-meta { padding-right: 0; } } .singular #content,.left-sidebar.singular #content { margin: 0 10%; width: 80%; } .singular .entry-header,.singular .entry-content,.singular footer.entry-meta,.singular #comments-title { width: 100%; } .hentry,.no-results { border-bottom: 1px dotted #dadada; } .singular .hentry { padding-top: 1em; } .entry-title { padding-bottom: 0; } .entry-title,.entry-title a { font-size: 28px; color: #000; } .entry-title a:hover,.entry-title a:focus,.entry-title a:active { color: #0063dc; background-color: transparent; } @media screen and (max-width: 650px) { .singular .entry-title { padding-top: 0; line-height: 42px; } } .singular .entry-content { margin-top: 0; } #jp-post-flair { margin-top: 3em; } footer.entry-meta .cat-links,footer.entry-meta .sep,footer.entry-meta .tag-links { display: none; } .singular .entry-meta .edit-link a { position: static; } .singular footer.entry-meta { margin-top: 2em; } #comments { display: none; } #site-generator { background: transparent; border-top: none; padding: 0 .5em; } img#wpstats { position: absolute !important; clip: rect(1px 1px 1px 1px); } #secondary { margin-right: 0; width: 16%; text-align: right; } #secondary #s { display: none; } #secondary aside ul { list-style: none; } #secondary .widget-title { color: #999; font-size: 12px; letter-spacing: auto; text-transform: none; } .widget a { font-weight: normal; } .widget a:hover,.widget a:focus,.widget a:active { text-decoration: none; } #everything-after-this-line-is-a-post-level-style---yay-for-wordpress-stripping-comments-from-css { color: #000; } .entry-content p { margin-bottom: 1.3em; } .entry-content p.note { padding: 11px; background-color: #fffdeb; border-bottom: 1px solid #fff9c2; } .entry-content p.warning { padding: 11px; background-color: #fdf8f8; border-bottom: 1px solid #f7dedd; } .entry-content .aside { margin-bottom: 1.3em; padding: 11px; background-color: #f8fdf8; border-bottom: 1px solid #def6df; } .entry-content .aside p:last-child { margin-bottom: 0; } .entry-content h2,.entry-content h3,.entry-content h4 { font-weight: bold; color: #000; text-transform: none; } .entry-content h2 { font-size: 22px; } .entry-content h3 { font-size: 16px; letter-spacing: 0; line-height: 1.3em; margin-bottom: 1em; } .entry-content h4 { font-size: 13px; line-height: 1.4em; margin-bottom: 0; } .entry-content p.flickr-photo,.entry-content p.flickr-photo a,.entry-content p.figure { color: #999; font-size: 14px; } .entry-content p.flickr-photo a:hover,.entry-content p.flickr-photo a:active { color: #0063dc; background-color: transparent; } .entry-content p.flickr-photo .caption { display: block; padding-left: 22px; background: url('http://farm4.staticflickr.com/3329/favicons/72157601614001242_7730.png') no-repeat 0 3px; } .entry-content img { max-width: 100%; height: auto; } .undersized-image-container { text-align: center; } .entry-content code { color: #000; font-family: "Consolas", "Bitstream Vera Sans Mono", "Courier New", Courier, monospace; font-size: 14px; background-color: #F6FcFF; border-bottom: 1px solid #ebf5ff; } .entry-content ul { list-style: disc; } .entry-content ol ol { list-style: lower-alpha; } .entry-content table caption { font-size: 13px; font-weight: bold; color: #000; } .entry-content table tr.odd td,.entry-content table tr.odd th { background-color: #f8f8f8; } .entry-content table td,.entry-content table th { padding: 3px 10px; line-height: 1.4em; } .entry-content table tr th { color: #000; font-weight: bold; text-transform: none; font-size: 13px; letter-spacing: 0; border-top: 1px solid #ddd; } .entry-content table tr td { font-size: 14px; } .entry-content table.data tr td,.entry-content table.data thead tr th { text-align: right; } .entry-content table.tight td,.entry-content table.tight th { padding: 3px 6px; } .entry-content table.tight th { font-size: 12px; } .entry-content table.tight td { font-size: 13px; } @media screen and (max-width: 650px) { .entry-content table td,.entry-content table th { padding: 3px 5px; } .entry-content table tr th { font-size: 12px; } .entry-content table tr td { font-size: 13px; } .entry-content table.tight td,.entry-content table.tight th { padding: 3px 4px; } .entry-content table.tight th { font-size: 11px; } .entry-content table.tight td { font-size: 12px; } } .entry-content blockquote { font-family: inherit; font-style: normal; font-weight: normal; margin: 0 2em 1.3em; padding: 1em; background: #F9F9F9; border-bottom: 1px solid #ececec; } .entry-content blockquote p.source { margin-bottom: 0; } .entry-content .hiring-banner { position: relative; background: #ffebf5; padding: 11px 11px 11px 88px; border-bottom: 1px solid #ffc5e2; margin: 2em 0; } .entry-content .hiring-banner p { margin-bottom: 0; } .entry-content .hiring-banner .group-photo { position: absolute; left: -57px; top: -5px; width: 120px; padding: 6px; background: #fff; -webkit-box-shadow: 1px 1px 8px rgba(50,50,50,0.8); -moz-box-shadow: 1px 1px 8px rgba(50,50,50,0.8); box-shadow: 1px 1px 8px rgba(50,50,50,0.8); -webkit-transform: rotate(-11deg); -moz-transform: rotate(-11deg); -o-transform: rotate(-11deg); -ms-transform: rotate(-11deg); } .entry-content .hiring-banner .group-photo img { display: block; margin: 0; max-width: 100%; } @media screen and (max-width: 800px) { .entry-content .hiring-banner { padding-left: 11px; } .entry-content .hiring-banner .group-photo { display: none; } } #everything-after-this-line-is-for-the-syntaxhighlighter---all-rules-require-important { color: #000; } #main .syntaxhighlighter .lines { border-bottom: 1px solid #ebf5ff !important; } #main .syntaxhighlighter,#main .syntaxhighlighter div,#main .syntaxhighlighter code,#main .syntaxhighlighter table,#main .syntaxhighlighter table td,#main .syntaxhighlighter table tr,#main .syntaxhighlighter table tbody { font-size: 14px !important; } #main .syntaxhighlighter .line.alt1,#main .syntaxhighlighter .line.alt2 { background-color: #fafdff !important; } #main .syntaxhighlighter .line.highlighted { background-color: #fffbd6 !important; } #main .syntaxhighlighter .plain,#main .syntaxhighlighter .plain a { color: #000 !important; } #main .syntaxhighlighter .comments,#main .syntaxhighlighter .comments a { color: #999 !important; } #main .syntaxhighlighter .string,#main .syntaxhighlighter .string a { color: #ff52a9 !important; } #main .syntaxhighlighter .keyword { color: #0034fe !important; font-weight: normal !important; } #main .syntaxhighlighter .preprocessor { color: #417ba9 !important; } #main .syntaxhighlighter .variable { color: #b130c0 !important; } #main .syntaxhighlighter .value { color: #6b77f7 !important; } #main .syntaxhighlighter .functions { color: #002ad5 !important; } #main .syntaxhighlighter .constants { color: #d11e08 !important; } #main .syntaxhighlighter .script { background-color: yellow !important; } #main .syntaxhighlighter .color1,#main .syntaxhighlighter .color1 a { color: #808080 !important; } #main .syntaxhighlighter .color2,#main .syntaxhighlighter .color2 a { color: #ff1493 !important; } #main .syntaxhighlighter .color3,#main .syntaxhighlighter .color3 a { color: red !important; } a[data-flickr-embed] img { width: 800px; } </style> </head> <body class="archive date custom-background wp-embed-responsive jps-theme-flickr-code two-column right-sidebar"> <div class="skip-link"><a class="assistive-text" href="#content">Skip to primary content</a></div><div class="skip-link"><a class="assistive-text" href="#secondary">Skip to secondary content</a></div><div id="page" class="hfeed"> <header id="branding"> <hgroup> <h1 id="site-title"><span><a href="https://code.flickr.net/" rel="home">code.flickr.com</a></span></h1> <h2 id="site-description"></h2> </hgroup> <a href="https://code.flickr.net/"> <img src="https://wp.flickr.net/wp-content/uploads/sites/3/2012/09/code-flickr-com-drawn-header-grey-large.png" width="1000" height="157" alt="code.flickr.com" decoding="async" fetchpriority="high" /> </a> <div class="only-search with-image"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> </form> </div> <nav id="access"> <h3 class="assistive-text">Main menu</h3> <div class="menu-menu-container"><ul id="menu-menu" class="menu"><li id="menu-item-2084" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2084"><a href="http://www.flickr.com/">Flickr</a></li> <li id="menu-item-2085" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2085"><a href="http://blog.flickr.net/">Flickr Blog</a></li> <li id="menu-item-2250" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2250"><a href="http://twitter.com/flickr">@flickr</a></li> <li id="menu-item-2086" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2086"><a href="http://twitter.com/flickrapi">@flickrapi</a></li> <li id="menu-item-2087" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2087"><a href="https://www.flickr.com/services/developer/">Developer Guidelines</a></li> <li id="menu-item-2088" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2088"><a href="http://www.flickr.com/services/api/">API</a></li> <li id="menu-item-2089" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2089"><a href="http://www.flickr.com/jobs/">Jobs</a></li> </ul></div> </nav> </header> <div id="main"> <section id="primary"> <div id="content" role="main"> <header class="page-header"> <h1 class="page-title"> Monthly Archives: <span>July 2014</span> </h1> </header> <article id="post-2647" class="post-2647 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2014/07/31/redis-sentinel-at-flickr/" rel="bookmark">Redis Sentinel at Flickr</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2014/07/31/redis-sentinel-at-flickr/" title="3:47 pm" rel="bookmark"><time class="entry-date" datetime="2014-07-31T15:47:59-07:00">July 31, 2014</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/sfncook/" title="View all posts by sfncook" rel="author">sfncook</a></span></span> </div> </header> <div class="entry-content"> <p class="flickr-photo"><a href="https://www.flickr.com/photos/93570740@N07/14804076425/" title="HIGH FIVE! by richard_thorn, on Flickr"><img decoding="async" src="https://farm4.staticflickr.com/3862/14804076425_01a6b1ab85_b.jpg" width="890" height="792" alt="HIGH FIVE!" border="0" /></a></p> <p style="text-align:justify;">We recently implemented Redis Sentinel at Flickr to provide automated Redis master failover for an important subsystem and we wanted to share our experience with it. Hopefully, we can provide insight into our experience adopting this relatively new technology and some of the nuances we encountered getting it up and running. Although we try to provide a basic explanation of what Sentinel is and how it works, anyone who is new to Redis or Sentinel should start with the excellent <a href="http://redis.io/documentation">Redis</a> and <a href="http://redis.io/topics/sentinel">Sentinel</a> documentation.</p> <p style="text-align:justify;">At Flickr we use an offline task processing system that allows us to execute heavyweight operations asynchronously from our API and web pages. This prevents these operations from making users wait needlessly for pages to render or API methods to return. Our task system handles millions of tasks per day which includes operations like <a href="https://www.flickr.com/photos/upload/">photo uploads</a>, user notifications and metadata edits. In this system, code can push a task onto one of several Redis-backed queues based on priority and operation, then forget about the task. Many of these operations are critical and we need to make sure we process at least 99.9999% of them (less than 1 in 1 million dropped). Additionally, we need to make sure this system is available to insert and process tasks at least 99.995% of the time – no more than about 2 minutes a month downtime.</p> <p>Until a few months ago our Redis BCP consisted of:</p> <ul> <li>Redis <a href="http://redis.io/topics/persistence"><i>append-only-files</i></a> (AOF)</li> <li>Warm-swappable <a href="http://redis.io/topics/replication">slaves</a></li> <li>Logging task insert failures</li> </ul> <p style="text-align:justify;">Upon master failure, the recovery plan included several manual steps: reconfiguring code to take the Redis master(s) offline and manually promoting a Redis slave (a mildly time consuming activity). Then we would rebuild and backfill unprocessed data from AOF files and error logs — a very time consuming activity. We knew if we lost a master we would have hours and hours of less-than-satisfying work to run the recovery plan, and there was potential for user impact and even a small amount of data loss. We had never experienced a Redis master failure, but we all know that such events are simply a matter of time. Overall, this fell far short of our durability and availability goals.</p> <h2><b>Configuring Sentinel</b></h2> <p style="text-align:justify;">We started by installing and testing Sentinel in a development environment and the first thing we noticed was how simple Sentinel is to use and how similar the syntax is to Redis. We read <a href="http://aphyr.com/posts/283-call-me-maybe-redis">Aphyr’s article</a> and his <a href="http://antirez.com/news/55">back-and-forth blog duel with Salvatore</a> and verified Aphyr’s warning about the “split brain” scenario. Eventually we decided the benefits outweighed the risks in our specific use case. During testing we learned about some Sentinel nuances and got a better feel for appropriate configuration values, many of which have little or no community guidance yet.</p> <p style="text-align:justify;">One such example was choosing a good value for the <a href="http://redis.io/topics/sentinel">level-of-agreement</a> setting, which is the number of Sentinels simultaneously reporting a host outage before automatic failover starts. If this value is too high then you’ll miss real failures and if it’s too low you are more susceptible to false alarms. (*thanks to Aleksey Asiutin(@aasiutin) for the edit!) In the end, we looked at the physical topology of our hosts over rack and switches and chose to run a relatively large number of Sentinel instances to ensure good coverage. Based on tuning in production we chose a value for level-of-agreement equal to about 80% of the Sentinel instances.</p> <p style="text-align:justify;">The <i>down-after-milliseconds</i> configuration setting is the time the Sentinels will wait with no response to their ping requests before declaring a host outage. Sentinels ping the hosts they monitor approximately every second, so by choosing a value of <i>3,100</i> we expect Sentinels to miss 3 pings before declaring host outage. Interestingly, because of Sentinel’s ping frequency we found that setting this value to less than <i>1,000 </i>results in an endless stream of host outage notifications from the Sentinels, so don’t do that. We also added an extra 100 milliseconds (3,100ms rather than 3,000ms) to allow for some variation in Redis response time.</p> <p style="text-align:justify;">We chose a <i>parallel-syncs</i> value of 1. This item dictates the number of slaves that are reconfigured simultaneously after a failover event. If you serve queries from the read-only slaves you’ll want to keep this value low.</p> <p style="text-align:justify;">For an explanation of the other values we refer you to the self-documented default <a href="http://download.redis.io/redis-stable/sentinel.conf">sentinel.conf</a> file.</p> <p style="text-align:justify;">An example of the Sentinel configuration we use:</p> <pre class="brush: plain; gutter: false; title: ; notranslate" title=""> sentinel monitor cluster_name_1 redis_host_1 6390 35 sentinel down-after-milliseconds cluster_name_1 3100 sentinel parallel-syncs cluster_name_1 1 sentinel monitor cluster_name_2 redis_host_2 6391 35 sentinel down-after-milliseconds cluster_name_2 3100 sentinel parallel-syncs cluster_name_2 1 port 26379 pidfile [path]/redis-sentinel.pid logfile [path]logs/redis/redis-sentinel.log daemonize yes </pre> <p style="text-align:justify;">An interesting nuance of Sentinels is that they write state to their configuration file. This presented a challenge for us because it conflicted with our change management procedures. How do we maintain a dependably consistent startup configuration if the Sentinels are modifying the config files at runtime? Our solution was to create two Sentinel config files. One is strictly maintained in Git and not modified by Sentinel. This “permanent” config file is part of our deployment process and is installed whenever we update our Sentinel system configuration (i.e.: “rarely”). We then wrote a startup script that first duplicates the “permanent” config file to a writable “temporary” config file, then starts Sentinel and passes it the “temporary” file via command-line params. Sentinels are allowed to modify the “temporary” files as they please.</p> <p><a href="https://www.flickr.com/photos/93570740@N07/14617454808/" title="sunset by richard_thorn, on Flickr"><img decoding="async" src="https://farm6.staticflickr.com/5582/14617454808_af4f4ed4ca_b.jpg" width="1024" height="686" alt="sunset" border="0" /></a></p> <h2>Interfacing with Sentinel</h2> <p style="text-align:justify;">A common misconception about Sentinel is that it resides in-band between Redis and Redis clients. In fact, Sentinel is out-of-band and is only contacted by your services on startup. Sentinel then publishes notifications when it detects a Redis outage. Your services subscribe to Sentinel, receive the initial Redis host list, and then carry on normal communication directly with the Redis host.</p> <p style="text-align:justify;">The Sentinel command syntax is very similar to Redis command syntax. Since Flickr has been using Redis for a long time the adaptation of pre-existing code was pretty straightforward for us. Code modifications consisted of adding a few Java classes and modifying our configuration syntax. For Java-to-Redis interaction we use <a href="http://redis.io/clients">Jedis</a>, and for PHP we use <a href="https://github.com/nrk/predis">Predis</a> and <a href="https://github.com/toymachine/libredis">libredis</a>.</p> <p style="text-align:justify;">Using Sentinel from Jedis is not documented as well as it could be. Here’s some code that we hope will save you some time:</p> <pre class="brush: java; gutter: false; title: ; notranslate" title=""> // Verify that at least one Sentinel instance in the Set is available and responding. // sentinelHostPorts: String format: [hostname]:[port] private boolean jedisSentinelPoolAvailable(Set&lt;String&gt; sentinelHostPorts, String clusterName){ log.info(&quot;Trying to find master from available Sentinels...&quot;); for ( String sentinelHostPort : sentinelHostPorts ) { List&lt;String&gt; hostPort = Arrays.asList( sentinelHostPort.split(&quot;:&quot;) ); String hostname = hostPort.get(0); int port = Integer.parseInt( hostPort.get(1) ); try { Jedis jedis = new Jedis( hostname, port ); jedis.sentinelGetMasterAddrByName( clusterName ); jedis.disconnect(); log.info(&quot;Connected to Sentinel host:%s port:%d&quot;, hostname, port); return true; } catch (JedisConnectionException e) { log.warn(&quot;Cannot connect to Sentinel host:%s port:%d”, hostname, port); } } return false; } private Pool&lt;Jedis&gt; getDefaultJedisPool() { // Create and return a default Jedis Pool object… // ... } // ConfigurationMgr configMgr ⇐ your favorite way of managing system configuration (up to you) public Pool&lt;Jedis&gt; getPool(ConfigurationMgr configMgr) { String clusterName = configMgr.getRedisClusterName(); Set&lt;String&gt; sentinelHostPorts = configMgr.getSentinelHostPorts(); if(sentinels.size()&gt;0) { if(jedisSentinelPoolAvailable( sentinelHostPorts, clusterName )) { return new JedisSentinelPool(clusterName, sentinelHostPorts); } else { log.warn(“All Sentinels unreachable. Using default Redis hosts.”); return getDefaultJedisPool(); } } else { log.warn(“Sentinel config empty. Using default Redis hosts.”); return getDefaultJedisPool(); } } </pre> <p class="figure"><a href="https://www.flickr.com/photos/93570740@N07/14617358400/" title="running with the bulls by richard_thorn, on Flickr"><img loading="lazy" decoding="async" src="https://farm3.staticflickr.com/2909/14617358400_1ef74f2731_b.jpg" width="1000" height="750" alt="running with the bulls" border="0" /></a></p> <h2>Testing Sentinel at Flickr</h2> <p style="text-align:justify;">Before deploying Sentinel to our production system we had several questions and concerns:</p> <ul style="text-align:justify;"> <li>How will the system react to host outages?</li> <li>How long does a failover event last?</li> <li>How much of a threat is the split-brain scenario?</li> <li>How much data loss can we expect from a failover?</li> </ul> <p style="text-align:justify;">We commandeered several development machines and installed a few Redis and Sentinel instances. Then we wrote some scripts that insert or remove data from Redis to simulate production usage.</p> <p><a href="https://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-19-56-pm.png"><img loading="lazy" decoding="async" class="wp-image-2708 aligncenter" src="http://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-19-56-pm.png?w=300" alt="Screen Shot 2014-07-23 at 3.19.56 PM" width="481" height="325" /></a></p> <p style="text-align:justify;">We ran a series of tests on this setup, simulating a variety of Redis host failures with some combination of the commands: <i>kill -9</i>, the <a href="http://redis.io/topics/sentinel">Sentinel <i>failover</i> command</a>, and Linux <i>iptables</i>. This resulted in “breaking” the system in various ways.</p> <p style="text-align:center;"><a href="http://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-20-16-pm.png"><img loading="lazy" decoding="async" class="wp-image-2709 aligncenter" src="http://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-20-16-pm.png?w=300" alt="Screen Shot 2014-07-23 at 3.20.16 PM" width="500" height="239" /></a> Figure: Redis master failure</p> <p style="text-align:left;"><a href="http://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-20-41-pm.png"><img loading="lazy" decoding="async" class="wp-image-2710 aligncenter" src="http://wp.flickr.net/wp-content/uploads/sites/3/2014/07/screen-shot-2014-07-23-at-3-20-41-pm.png?w=300" alt="Screen Shot 2014-07-23 at 3.20.41 PM" width="533" height="286" /></a></p> <p style="text-align:center;">Figure: Network partition producing a ‘split-brain’ scenario</p> <h3 style="text-align:justify;"><strong><b>How will the system react to host outages?</b></strong></h3> <p style="text-align:justify;">For the most part we found Sentinel to behave exactly as expected and described in the <a href="http://redis.io/topics/sentinel">Sentinel docs</a>. The Sentinels detect host outages within the configured <i>down-after-milliseconds</i> duration, then send “subjective down” notifications, then send “objective down” notifications if the <i>level-of-agreement</i> threshold is reached. In this environment we were able to quickly and accurately test our response to failover events. We began with small test scripts, but eventually were able to run repeatable integration tests on our production software. Adding Redis to a Maven test phase for automated integration testing is a backlog item that we haven’t implemented yet.</p> <h3 style="text-align:justify;"><strong><b>How long does a failover event last?</b></strong></h3> <p style="text-align:justify;">The Sentinel test environment was configured with a <i>down-after-milliseconds</i> value of 3,100ms (just like production, see above). With this value Sentinels would produce a host outage notification after approximately 3 unsuccessful pings (one ping per second). In addition to the 3,100ms delay, we found there were 1-3 seconds in overhead for processing the failover event and electing a new Redis master, resulting in 4-6 seconds of total downtime. We are pretty confident we’ll see the same behavior in production (verified — see below).</p> <h3 style="text-align:justify;"><strong><b>How much of a threat is the “split-brain” scenario?</b></strong></h3> <p style="text-align:justify;">We carefully read <a href="http://aphyr.com/posts/283-call-me-maybe-redis">Aphyr</a> and <a href="http://antirez.com/news/55%20">Salvatore’s</a> blog articles debating the threat of a “split brain scenario.” To summarize: this is a situation in which network connectivity is split, with some nodes still functioning on one side and other nodes continuing to function independently on the other side. The concern is the potential for the data set to diverge with different data being written to masters on both sides of the partition. This could easily create data that is either impossible or very difficult to reconcile.</p> <p style="text-align:justify;">We recreated this situation and verified that a network partition could create disjoint concurrent data sets. Removing the partition resulted in Sentinel arbitrarily (from our perspective) choosing a new master and losing all data written (post-partitioning) to the other master. So the question is: given our production architecture, what is the probability of this happening and is it acceptable given the significant benefit of automatic failover?</p> <p style="text-align:justify;">We looked at this scenario in detail considering all the potential failure modes in our deployment. Although we believe our production environment is not immune from split-brain, we are pretty sure that the benefits outweigh the risks.</p> <h3 style="text-align:justify;"><strong><b>How much data loss can we expect from a failover event?</b></strong></h3> <p style="text-align:justify;">After testing we were confident that Redis host outages could produce 4-6 seconds of downtime in this system. Rapid Sentinel automated failover events combined with reasonable backoff and retry techniques in the code logic were expected to further reduce data loss during a failover event. With Sentinel deployed and considering a long history of a highly stable Redis operation, we believed we could achieve 99.995% or more production availability – a few minutes of downtime per year.</p> <h2 style="text-align:justify;"><strong><b>Sentinel in Production</b></strong></h2> <p style="text-align:justify;">So how has Sentinel performed in production? Mostly it has been silent, which is a good thing. A month after finishing our deployment we had a hardware failure in a network switch that had some of our Redis masters behind it. Instead of having a potential scenario involving tens of minutes of user impact with human-in-the-loop actions to restore service, automatic failover allowed us to limit impact to just seconds with no human intervention. Due to the quick master failover and other reliability features in the code, only 270 tasks failed to insert due to the outage — all of which were captured by logging. Based on the volume of tasks in the system, this met our 99.9999% task durability goal. We did however decide to re-run a couple tasks manually and for certain critical and low-volume tasks we’re looking at providing even more reliability.</p> <p style="text-align:justify;">One more note from production experience. We occasionally see Sentinels reporting false “subjective down” events. Our Sentinel instances cohabitate host machines with other services. Occasionally these hosts get busy and we suspect these occasional load spikes affect the Sentinels’ ability to send and receive ping requests. But because our level-of-agreement is high, these false alarms do not trigger objective down events and are relatively harmless. If you’re deploying Sentinel on hosts that share other workloads, make sure that you consider potential impact of load patterns on those hosts and make sure you take some time to tune your level-of-agreement.</p> <h2 style="text-align:justify;"><strong><b>Conclusion</b></strong></h2> <p style="text-align:justify;">We have been very happy with Sentinel’s ease of use, relatively simple learning curve and brilliant production execution. So far the Redis/Sentinel combination is working great for us.</p> <div class="hiring-banner"> <p class="group-photo"> <a href="http://www.flickr.com/photos/morozgrafix/7803402076/" title="Flickr flamily floto by morozgrafix, on Flickr"><img loading="lazy" decoding="async" src="http://farm9.staticflickr.com/8436/7803402076_c06f87bf1f_m.jpg" width="120" height="80" alt="Flickr flamily floto"></a> </p> <p> Like this post? Have a love of online photography? Want to work with us? Flickr is hiring <strong>engineers</strong>, <strong>designers</strong> and <strong>product managers</strong> in our San Francisco office. <strong>Find out more at <a href="http://www.flickr.com/jobs/">flickr.com/jobs</a></strong>. </p> </div> <h2 style="text-align:justify;"><strong><b>References</b></strong></h2> <ol style="text-align:justify;"> <li>Redis Sentinel Documentation – <a href="http://redis.io/topics/sentinel">http://redis.io/topics/sentinel</a></li> <li>Redis Command Reference – <a href="http://redis.io/commands">http://redis.io/commands</a></li> <li>Aphyr: Call me maybe: Redis – <a href="http://aphyr.com/posts/283-call-me-maybe-redis">http://aphyr.com/posts/283-call-me-maybe-redis</a></li> <li>Antirez: Reply to Aphyr Attack on Sentinel – <a href="http://antirez.com/news/55">http://antirez.com/news/55</a></li> <li>Jedis Project Page – <a href="https://github.com/xetorthio/jedis">https://github.com/xetorthio/jedis</a></li> </ol> <p style="text-align:justify;"> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> </div> </section> <div id="secondary" class="widget-area" role="complementary"> <aside id="jetpack-search-filters-3" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-3-wrapper" class="jetpack-instant-search-wrapper"> <div class="jetpack-search-form"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> <input type="hidden" name="orderby" value="" /><input type="hidden" name="order" value="" /></form> </div> <h4 class="jetpack-search-filters-widget__sub-heading"> Categories </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uncategorized"> Uncategorized (136) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="geo"> geo (12) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="kittens"> kittens (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="change-log"> changelog (7) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uploadr"> uploadr (6) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Tags </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="api"> api (24) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="geo"> geo (13) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="machine-tags"> machine tags (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="javascript"> javascript (9) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="kittentuesday"> kittentuesday (8) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Year </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2022-01-01 00:00:00" > 2022 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2021-01-01 00:00:00" > 2021 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2018-01-01 00:00:00" > 2018 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2017-01-01 00:00:00" > 2017 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2016-01-01 00:00:00" > 2016 (5) </a> </li> </ul> </div></aside> <aside id="recent-posts-2" class="widget widget_recent_entries"> <h3 class="widget-title">Recent Posts</h3> <ul> <li> <a href="https://code.flickr.net/2022/02/14/safer-internet-day-and-open-source-codes-of-conduct/">Safer Internet Day and Open Source Codes of Conduct</a> </li> <li> <a href="https://code.flickr.net/2022/01/04/a-pluggable-solution-for-api-observability-on-our-php-system/">A Pluggable Solution for API Observability on our PHP System</a> </li> <li> <a href="https://code.flickr.net/2021/11/22/flickr-engineering-team-vision-guiding-principles/">Flickr Engineering Team Vision & Guiding Principles</a> </li> <li> <a href="https://code.flickr.net/2018/04/20/together/">Together</a> </li> <li> <a href="https://code.flickr.net/2017/03/07/introducing-similarity-search-at-flickr/">Introducing Similarity Search at Flickr</a> </li> </ul> </aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3> <ul> <li><a href='https://code.flickr.net/2022/02/'>February 2022</a></li> <li><a href='https://code.flickr.net/2022/01/'>January 2022</a></li> <li><a href='https://code.flickr.net/2021/11/'>November 2021</a></li> <li><a href='https://code.flickr.net/2018/04/'>April 2018</a></li> <li><a href='https://code.flickr.net/2017/03/'>March 2017</a></li> <li><a href='https://code.flickr.net/2017/01/'>January 2017</a></li> <li><a href='https://code.flickr.net/2016/09/'>September 2016</a></li> <li><a href='https://code.flickr.net/2016/05/'>May 2016</a></li> <li><a href='https://code.flickr.net/2016/04/'>April 2016</a></li> <li><a href='https://code.flickr.net/2016/03/'>March 2016</a></li> <li><a href='https://code.flickr.net/2015/12/'>December 2015</a></li> <li><a href='https://code.flickr.net/2015/11/'>November 2015</a></li> <li><a href='https://code.flickr.net/2015/09/'>September 2015</a></li> <li><a href='https://code.flickr.net/2015/07/'>July 2015</a></li> <li><a href='https://code.flickr.net/2015/06/'>June 2015</a></li> <li><a href='https://code.flickr.net/2015/03/'>March 2015</a></li> <li><a href='https://code.flickr.net/2014/10/'>October 2014</a></li> <li><a href='https://code.flickr.net/2014/08/'>August 2014</a></li> <li><a href='https://code.flickr.net/2014/07/' aria-current="page">July 2014</a></li> <li><a href='https://code.flickr.net/2014/05/'>May 2014</a></li> <li><a href='https://code.flickr.net/2014/04/'>April 2014</a></li> <li><a href='https://code.flickr.net/2014/02/'>February 2014</a></li> <li><a href='https://code.flickr.net/2013/09/'>September 2013</a></li> <li><a href='https://code.flickr.net/2013/06/'>June 2013</a></li> <li><a href='https://code.flickr.net/2013/03/'>March 2013</a></li> <li><a href='https://code.flickr.net/2012/12/'>December 2012</a></li> <li><a href='https://code.flickr.net/2012/10/'>October 2012</a></li> <li><a href='https://code.flickr.net/2012/07/'>July 2012</a></li> <li><a href='https://code.flickr.net/2012/06/'>June 2012</a></li> <li><a href='https://code.flickr.net/2012/05/'>May 2012</a></li> <li><a href='https://code.flickr.net/2012/04/'>April 2012</a></li> <li><a href='https://code.flickr.net/2012/02/'>February 2012</a></li> <li><a href='https://code.flickr.net/2012/01/'>January 2012</a></li> <li><a href='https://code.flickr.net/2011/12/'>December 2011</a></li> <li><a href='https://code.flickr.net/2011/10/'>October 2011</a></li> <li><a href='https://code.flickr.net/2011/09/'>September 2011</a></li> <li><a href='https://code.flickr.net/2011/08/'>August 2011</a></li> <li><a href='https://code.flickr.net/2011/07/'>July 2011</a></li> <li><a href='https://code.flickr.net/2011/06/'>June 2011</a></li> <li><a href='https://code.flickr.net/2011/03/'>March 2011</a></li> <li><a href='https://code.flickr.net/2011/02/'>February 2011</a></li> <li><a href='https://code.flickr.net/2011/01/'>January 2011</a></li> <li><a href='https://code.flickr.net/2010/11/'>November 2010</a></li> <li><a href='https://code.flickr.net/2010/10/'>October 2010</a></li> <li><a href='https://code.flickr.net/2010/09/'>September 2010</a></li> <li><a href='https://code.flickr.net/2010/08/'>August 2010</a></li> <li><a href='https://code.flickr.net/2010/07/'>July 2010</a></li> <li><a href='https://code.flickr.net/2010/05/'>May 2010</a></li> <li><a href='https://code.flickr.net/2010/04/'>April 2010</a></li> <li><a href='https://code.flickr.net/2010/03/'>March 2010</a></li> <li><a href='https://code.flickr.net/2010/02/'>February 2010</a></li> <li><a href='https://code.flickr.net/2010/01/'>January 2010</a></li> <li><a href='https://code.flickr.net/2009/12/'>December 2009</a></li> <li><a href='https://code.flickr.net/2009/11/'>November 2009</a></li> <li><a href='https://code.flickr.net/2009/10/'>October 2009</a></li> <li><a href='https://code.flickr.net/2009/09/'>September 2009</a></li> <li><a href='https://code.flickr.net/2009/07/'>July 2009</a></li> <li><a href='https://code.flickr.net/2009/06/'>June 2009</a></li> <li><a href='https://code.flickr.net/2009/05/'>May 2009</a></li> <li><a href='https://code.flickr.net/2009/04/'>April 2009</a></li> <li><a href='https://code.flickr.net/2009/03/'>March 2009</a></li> <li><a href='https://code.flickr.net/2009/02/'>February 2009</a></li> <li><a href='https://code.flickr.net/2009/01/'>January 2009</a></li> <li><a href='https://code.flickr.net/2008/12/'>December 2008</a></li> <li><a href='https://code.flickr.net/2008/11/'>November 2008</a></li> <li><a href='https://code.flickr.net/2008/10/'>October 2008</a></li> <li><a href='https://code.flickr.net/2008/09/'>September 2008</a></li> <li><a href='https://code.flickr.net/2008/08/'>August 2008</a></li> <li><a href='https://code.flickr.net/2008/07/'>July 2008</a></li> <li><a href='https://code.flickr.net/2008/06/'>June 2008</a></li> <li><a href='https://code.flickr.net/2008/05/'>May 2008</a></li> <li><a href='https://code.flickr.net/2008/04/'>April 2008</a></li> </ul> </aside><aside id="categories-2" class="widget widget_categories"><h3 class="widget-title">Categories</h3> <ul> <li class="cat-item cat-item-11749740"><a href="https://code.flickr.net/category/api-2/">API</a> </li> <li class="cat-item cat-item-564792"><a href="https://code.flickr.net/category/change-log/">changelog</a> </li> <li class="cat-item cat-item-5784"><a href="https://code.flickr.net/category/event/">event</a> </li> <li class="cat-item cat-item-29160"><a href="https://code.flickr.net/category/geo/">geo</a> </li> <li class="cat-item cat-item-139037766"><a href="https://code.flickr.net/category/hadoop/">hadoop</a> </li> <li class="cat-item cat-item-32"><a href="https://code.flickr.net/category/infrastructure/">infrastructure</a> </li> <li class="cat-item cat-item-139037765"><a href="https://code.flickr.net/category/kittens/">kittens</a> </li> <li class="cat-item cat-item-20156"><a href="https://code.flickr.net/category/labs/">labs</a> </li> <li class="cat-item cat-item-171"><a href="https://code.flickr.net/category/meta/">meta</a> </li> <li class="cat-item cat-item-7092"><a href="https://code.flickr.net/category/metrics/">metrics</a> </li> <li class="cat-item cat-item-139037764"><a href="https://code.flickr.net/category/open-source/">open source</a> </li> <li class="cat-item cat-item-1930"><a href="https://code.flickr.net/category/performance/">performance</a> </li> <li class="cat-item cat-item-304"><a href="https://code.flickr.net/category/photos/">photos</a> </li> <li class="cat-item cat-item-2373"><a href="https://code.flickr.net/category/search/">search</a> </li> <li class="cat-item cat-item-1"><a href="https://code.flickr.net/category/uncategorized/">Uncategorized</a> </li> <li class="cat-item cat-item-249276"><a href="https://code.flickr.net/category/uploadr/">uploadr</a> </li> <li class="cat-item cat-item-412"><a href="https://code.flickr.net/category/video/">video</a> </li> <li class="cat-item cat-item-830560"><a href="https://code.flickr.net/category/xulrunner/">xulrunner</a> </li> </ul> </aside><aside id="meta-2" class="widget widget_meta"><h3 class="widget-title">Meta</h3> <ul> <li><a href="https://code.flickr.net/wp-login.php">Log in</a></li> <li><a href="https://code.flickr.net/feed/">Entries feed</a></li> <li><a href="https://code.flickr.net/comments/feed/">Comments feed</a></li> <li><a href="https://wordpress.org/">WordPress.org</a></li> </ul> </aside> </div> </div> <footer id="colophon" role="contentinfo"> <div id="site-generator"> © 2025 Flickr, Inc. All rights reserved. | Powered by <a href="https://wpvip.com/?utm_source=vip_powered_wpcom&utm_medium=web&utm_campaign=VIP%20Footer%20Credit&utm_term=code.flickr.net" rel="generator nofollow" class="powered-by-wpcom">WordPress VIP</a> </div> </footer> </div> <div class="jetpack-instant-search__widget-area" style="display: none"> <div id="jetpack-search-filters-2" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-2-wrapper" class="jetpack-instant-search-wrapper"> </div></div> </div> <script type="text/javascript" src="https://code.flickr.net/_static/??-eJzTLy/QTc7PK0nNK9EvyClNz8wr1i+uzCtJrMjITM/IAeKS1CJMEWP94uSizIISoOIM5/yiVL2sYh19yo1yKiotzgjISczMo6aBXolliUDz7HNtDc1NDEwMjS0sDbMA2tRf4Q==" ></script><script type='text/javascript'> (function(){ var corecss = document.createElement('link'); var themecss = document.createElement('link'); var corecssurl = "https://code.flickr.net/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shCore.css?ver=3.0.9b"; if ( corecss.setAttribute ) { corecss.setAttribute( "rel", "stylesheet" ); corecss.setAttribute( "type", "text/css" ); corecss.setAttribute( "href", corecssurl ); } else { corecss.rel = "stylesheet"; corecss.href = corecssurl; } document.head.appendChild( corecss ); var themecssurl = "https://code.flickr.net/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shThemeDefault.css?ver=3.0.9b"; if ( themecss.setAttribute ) { themecss.setAttribute( "rel", "stylesheet" ); themecss.setAttribute( "type", "text/css" ); themecss.setAttribute( "href", themecssurl ); } else { themecss.rel = "stylesheet"; themecss.href = themecssurl; } document.head.appendChild( themecss ); })(); SyntaxHighlighter.config.strings.expandSource = '+ expand source'; SyntaxHighlighter.config.strings.help = '?'; SyntaxHighlighter.config.strings.alert = 'SyntaxHighlighter\n\n'; SyntaxHighlighter.config.strings.noBrush = 'Can\'t find brush for: '; SyntaxHighlighter.config.strings.brushNotHtmlScript = 'Brush wasn\'t configured for html-script option: '; SyntaxHighlighter.defaults['pad-line-numbers'] = false; SyntaxHighlighter.defaults['toolbar'] = false; SyntaxHighlighter.all(); // Infinite scroll support if ( typeof( jQuery ) !== 'undefined' ) { jQuery( function( $ ) { $( document.body ).on( 'post-load', function() { SyntaxHighlighter.highlight(); } ); } ); } </script> <script type="text/javascript" src="https://code.flickr.net/wp-includes/js/dist/hooks.min.js?m=1742994401g" ></script><script type="text/javascript" src="https://code.flickr.net/wp-includes/js/dist/i18n.min.js?ver=5e580eb46a90c2b997e6" id="wp-i18n-js"></script> <script type="text/javascript" id="wp-i18n-js-after"> /* <![CDATA[ */ wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } ); /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/wp-content/mu-plugins/jetpack-14.4/jetpack_vendor/automattic/jetpack-assets/build/i18n-loader.js?minify=true&ver=becd7d9884bc1b331e45" id="wp-jp-i18n-loader-js"></script> <script type="text/javascript" id="wp-jp-i18n-loader-js-after"> /* <![CDATA[ */ wp.jpI18nLoader.state = {"baseUrl":"https://code.flickr.net/wp-content/languages/","locale":"en_US","domainMap":{"jetpack-admin-ui":"plugins/jetpack","jetpack-assets":"plugins/jetpack","jetpack-backup-pkg":"plugins/jetpack","jetpack-blaze":"plugins/jetpack","jetpack-boost-core":"plugins/jetpack","jetpack-boost-speed-score":"plugins/jetpack","jetpack-classic-theme-helper":"plugins/jetpack","jetpack-compat":"plugins/jetpack","jetpack-config":"plugins/jetpack","jetpack-connection":"plugins/jetpack","jetpack-explat":"plugins/jetpack","jetpack-external-media":"plugins/jetpack","jetpack-forms":"plugins/jetpack","jetpack-image-cdn":"plugins/jetpack","jetpack-import":"plugins/jetpack","jetpack-ip":"plugins/jetpack","jetpack-jitm":"plugins/jetpack","jetpack-licensing":"plugins/jetpack","jetpack-masterbar":"plugins/jetpack","jetpack-my-jetpack":"plugins/jetpack","jetpack-password-checker":"plugins/jetpack","jetpack-plugins-installer":"plugins/jetpack","jetpack-post-list":"plugins/jetpack","jetpack-protect-models":"plugins/jetpack","jetpack-protect-status":"plugins/jetpack","jetpack-publicize-pkg":"plugins/jetpack","jetpack-search-pkg":"plugins/jetpack","jetpack-stats":"plugins/jetpack","jetpack-stats-admin":"plugins/jetpack","jetpack-sync":"plugins/jetpack","jetpack-videopress-pkg":"plugins/jetpack","jetpack-waf":"plugins/jetpack","jetpack-wordads":"plugins/jetpack","woocommerce-analytics":"plugins/jetpack"},"domainPaths":{"jetpack-admin-ui":"jetpack_vendor/automattic/jetpack-admin-ui/","jetpack-assets":"jetpack_vendor/automattic/jetpack-assets/","jetpack-backup-pkg":"jetpack_vendor/automattic/jetpack-backup/","jetpack-blaze":"jetpack_vendor/automattic/jetpack-blaze/","jetpack-boost-core":"jetpack_vendor/automattic/jetpack-boost-core/","jetpack-boost-speed-score":"jetpack_vendor/automattic/jetpack-boost-speed-score/","jetpack-classic-theme-helper":"jetpack_vendor/automattic/jetpack-classic-theme-helper/","jetpack-compat":"jetpack_vendor/automattic/jetpack-compat/","jetpack-config":"jetpack_vendor/automattic/jetpack-config/","jetpack-connection":"jetpack_vendor/automattic/jetpack-connection/","jetpack-explat":"jetpack_vendor/automattic/jetpack-explat/","jetpack-external-media":"jetpack_vendor/automattic/jetpack-external-media/","jetpack-forms":"jetpack_vendor/automattic/jetpack-forms/","jetpack-image-cdn":"jetpack_vendor/automattic/jetpack-image-cdn/","jetpack-import":"jetpack_vendor/automattic/jetpack-import/","jetpack-ip":"jetpack_vendor/automattic/jetpack-ip/","jetpack-jitm":"jetpack_vendor/automattic/jetpack-jitm/","jetpack-licensing":"jetpack_vendor/automattic/jetpack-licensing/","jetpack-masterbar":"jetpack_vendor/automattic/jetpack-masterbar/","jetpack-my-jetpack":"jetpack_vendor/automattic/jetpack-my-jetpack/","jetpack-password-checker":"jetpack_vendor/automattic/jetpack-password-checker/","jetpack-plugins-installer":"jetpack_vendor/automattic/jetpack-plugins-installer/","jetpack-post-list":"jetpack_vendor/automattic/jetpack-post-list/","jetpack-protect-models":"jetpack_vendor/automattic/jetpack-protect-models/","jetpack-protect-status":"jetpack_vendor/automattic/jetpack-protect-status/","jetpack-publicize-pkg":"jetpack_vendor/automattic/jetpack-publicize/","jetpack-search-pkg":"jetpack_vendor/automattic/jetpack-search/","jetpack-stats":"jetpack_vendor/automattic/jetpack-stats/","jetpack-stats-admin":"jetpack_vendor/automattic/jetpack-stats-admin/","jetpack-sync":"jetpack_vendor/automattic/jetpack-sync/","jetpack-videopress-pkg":"jetpack_vendor/automattic/jetpack-videopress/","jetpack-waf":"jetpack_vendor/automattic/jetpack-waf/","jetpack-wordads":"jetpack_vendor/automattic/jetpack-wordads/","woocommerce-analytics":"jetpack_vendor/automattic/woocommerce-analytics/"}}; /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/_static/??/wp-includes/js/dist/vendor/wp-polyfill.min.js,/wp-includes/js/dist/url.min.js?m=1742994401j" ></script><script type="text/javascript" id="jetpack-instant-search-js-before"> /* <![CDATA[ */ var JetpackInstantSearchOptions=JSON.parse(decodeURIComponent("%7B%22overlayOptions%22%3A%7B%22colorTheme%22%3A%22light%22%2C%22enableInfScroll%22%3Atrue%2C%22enableFilteringOpensOverlay%22%3Atrue%2C%22enablePostDate%22%3Atrue%2C%22enableSort%22%3Atrue%2C%22highlightColor%22%3A%22%23FFC%22%2C%22overlayTrigger%22%3A%22submit%22%2C%22resultFormat%22%3A%22expanded%22%2C%22showPoweredBy%22%3Atrue%2C%22defaultSort%22%3A%22relevance%22%2C%22excludedPostTypes%22%3A%5B%5D%7D%2C%22homeUrl%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%22%2C%22locale%22%3A%22en-US%22%2C%22postsPerPage%22%3A10%2C%22siteId%22%3A185426273%2C%22postTypes%22%3A%7B%22post%22%3A%7B%22singular_name%22%3A%22Post%22%2C%22name%22%3A%22Posts%22%7D%2C%22page%22%3A%7B%22singular_name%22%3A%22Page%22%2C%22name%22%3A%22Pages%22%7D%2C%22attachment%22%3A%7B%22singular_name%22%3A%22Media%22%2C%22name%22%3A%22Media%22%7D%7D%2C%22webpackPublicPath%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-content%5C%2Fmu-plugins%5C%2Fjetpack-14.4%5C%2Fjetpack_vendor%5C%2Fautomattic%5C%2Fjetpack-search%5C%2Fbuild%5C%2Finstant-search%5C%2F%22%2C%22isPhotonEnabled%22%3Afalse%2C%22isFreePlan%22%3Afalse%2C%22apiRoot%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-json%5C%2F%22%2C%22apiNonce%22%3A%226532796dd2%22%2C%22isPrivateSite%22%3Afalse%2C%22isWpcom%22%3Afalse%2C%22hasOverlayWidgets%22%3Atrue%2C%22widgets%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_0%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_1%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_2%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22date_histogram_3%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%7D%5D%2C%22widgetsOutsideOverlay%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_4%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_5%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_6%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22date_histogram_7%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%7D%5D%2C%22hasNonSearchWidgets%22%3Afalse%2C%22preventTrackingCookiesReset%22%3Afalse%7D")); /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/wp-content/mu-plugins/jetpack-14.4/jetpack_vendor/automattic/jetpack-search/build/instant-search/jp-search.js?minify=false&ver=749aa845a5ccb16bc8a6" id="jetpack-instant-search-js"></script> <script type="text/javascript" src="//stats.wp.com/w.js?ver=202514" id="jp-tracks-js"></script> <script type="text/javascript" id="jetpack-stats-js-before"> /* <![CDATA[ */ _stq = window._stq || []; _stq.push([ "view", JSON.parse("{\"v\":\"ext\",\"blog\":\"185426273\",\"post\":\"0\",\"tz\":\"-7\",\"srv\":\"code.flickr.net\",\"hp\":\"vip\",\"j\":\"1:14.4.1\"}") ]); _stq.push([ "clickTrackerInit", "185426273", "0" ]); /* ]]> */ </script> <script type="text/javascript" src="https://stats.wp.com/e-202514.js" id="jetpack-stats-js" defer="defer" data-wp-strategy="defer"></script> <script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script> </body> </html>