CINXE.COM

<!DOCTYPE html>     <html lang="en-US">  <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width" /> <title> September | 2015 | code.flickr.com </title> <link rel="profile" href="https://gmpg.org/xfn/11" /> <link rel="stylesheet" type="text/css" media="all" href="https://code.flickr.net/wp-content/themes/flickr-code/style.css?ver=20240716" /> <link rel="pingback" href="https://code.flickr.net/xmlrpc.php">  <meta name='robots' content='max-image-preview:large' /> <style>img:is([sizes="auto" i], [sizes^="auto," i]) { contain-intrinsic-size: 3000px 1500px }</style> <link rel='dns-prefetch' href='//stats.wp.com' /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Feed" href="https://code.flickr.net/feed/" /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Comments Feed" href="https://code.flickr.net/comments/feed/" /> <script type="text/javascript"> /* <![CDATA[ */ window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/15.0.3\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/code.flickr.net\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.7.2"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83d\udc26\u200d\u2b1b","\ud83d\udc26\u200b\u2b1b")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); /* ]]> */ </script> <style id='wp-emoji-styles-inline-css'> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='all-css-2' href='https://code.flickr.net/wp-includes/css/dist/block-library/style.min.css?m=1742994400g' type='text/css' media='all' /> <style id='wp-block-library-theme-inline-css'> .wp-block-audio :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-audio :where(figcaption){color:#ffffffa6}.wp-block-audio{margin:0 0 1em}.wp-block-code{border:1px solid #ccc;border-radius:4px;font-family:Menlo,Consolas,monaco,monospace;padding:.8em 1em}.wp-block-embed :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-embed :where(figcaption){color:#ffffffa6}.wp-block-embed{margin:0 0 1em}.blocks-gallery-caption{color:#555;font-size:13px;text-align:center}.is-dark-theme .blocks-gallery-caption{color:#ffffffa6}:root :where(.wp-block-image figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme :root :where(.wp-block-image figcaption){color:#ffffffa6}.wp-block-image{margin:0 0 1em}.wp-block-pullquote{border-bottom:4px solid;border-top:4px solid;color:currentColor;margin-bottom:1.75em}.wp-block-pullquote cite,.wp-block-pullquote footer,.wp-block-pullquote__citation{color:currentColor;font-size:.8125em;font-style:normal;text-transform:uppercase}.wp-block-quote{border-left:.25em solid;margin:0 0 1.75em;padding-left:1em}.wp-block-quote cite,.wp-block-quote footer{color:currentColor;font-size:.8125em;font-style:normal;position:relative}.wp-block-quote:where(.has-text-align-right){border-left:none;border-right:.25em solid;padding-left:0;padding-right:1em}.wp-block-quote:where(.has-text-align-center){border:none;padding-left:0}.wp-block-quote.is-large,.wp-block-quote.is-style-large,.wp-block-quote:where(.is-style-plain){border:none}.wp-block-search .wp-block-search__label{font-weight:700}.wp-block-search__button{border:1px solid #ccc;padding:.375em .625em}:where(.wp-block-group.has-background){padding:1.25em 2.375em}.wp-block-separator.has-css-opacity{opacity:.4}.wp-block-separator{border:none;border-bottom:2px solid;margin-left:auto;margin-right:auto}.wp-block-separator.has-alpha-channel-opacity{opacity:1}.wp-block-separator:not(.is-style-wide):not(.is-style-dots){width:100px}.wp-block-separator.has-background:not(.is-style-dots){border-bottom:none;height:1px}.wp-block-separator.has-background:not(.is-style-wide):not(.is-style-dots){height:2px}.wp-block-table{margin:0 0 1em}.wp-block-table td,.wp-block-table th{word-break:normal}.wp-block-table :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-table :where(figcaption){color:#ffffffa6}.wp-block-video :where(figcaption){color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-video :where(figcaption){color:#ffffffa6}.wp-block-video{margin:0 0 1em}:root :where(.wp-block-template-part.has-background){margin-bottom:0;margin-top:0;padding:1.25em 2.375em} </style> <link rel='stylesheet' id='all-css-6' href='https://code.flickr.net/_static/??-eJzTLy/QzcxLzilNSS3WzyrWz01NyUxMzUnNTc0rQeEU5CRWphbp5qSmJyZX6uVm5uklFxfr6OPTDpRD5sM02efaGpqbGFlampgYGAIAROAu5A==' type='text/css' media='all' /> <style id='jetpack-sharing-buttons-style-inline-css'> .jetpack-sharing-buttons__services-list{display:flex;flex-direction:row;flex-wrap:wrap;gap:0;list-style-type:none;margin:5px;padding:0}.jetpack-sharing-buttons__services-list.has-small-icon-size{font-size:12px}.jetpack-sharing-buttons__services-list.has-normal-icon-size{font-size:16px}.jetpack-sharing-buttons__services-list.has-large-icon-size{font-size:24px}.jetpack-sharing-buttons__services-list.has-huge-icon-size{font-size:36px}@media print{.jetpack-sharing-buttons__services-list{display:none!important}}.editor-styles-wrapper .wp-block-jetpack-sharing-buttons{gap:0;padding-inline-start:0}ul.jetpack-sharing-buttons__services-list.has-background{padding:1.25em 2.375em} </style> <style id='classic-theme-styles-inline-css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css'> :root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--blue: #1982d1;--wp--preset--color--dark-gray: #373737;--wp--preset--color--medium-gray: #666;--wp--preset--color--light-gray: #e2e2e2;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} :root :where(.wp-block-pullquote){font-size: 1.5em;line-height: 1.6;} </style> <link rel='stylesheet' id='all-css-12' href='https://code.flickr.net/_static/??-eJzTLy/QTc7PK0nNK9EvyUjNTS3WLykHcipTc1LLUvP0i0sqc1L1kouLdfQxVablZCZnFwFFU1LxK0QxMiknPzm7GKTUPtfW0NzEwMTQ2MLSEABabDMX' type='text/css' media='all' /> <link rel="https://api.w.org/" href="https://code.flickr.net/wp-json/" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://code.flickr.net/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.7.2" /> <style>img#wpstats{display:none}</style> <style type="text/css" id="twentyeleven-header-css"> #site-title, #site-description { position: absolute; clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ clip: rect(1px, 1px, 1px, 1px); } </style> <style type="text/css" id="wp-custom-css"> body { background: #fff; } body,input,textarea { font-family: Arial, sans-serif; color: #404040; } #page { margin: 1em auto; max-width: 1000px; } a:link { text-decoration: none; color: #0063dc; } a:visited { text-decoration: none; color: #0063dc; } a:hover { text-decoration: none; background: #0063dc; color: #fff; } a:active { text-decoration: none; background-color: #0259c4; color: #fff; } #branding { border: none; } #branding .only-search #s,#branding .only-search #s:focus { width: 280px; background-color: transparent; border-color: #ddd; } #branding #searchform { right: 0; } nav#access { background: none; box-shadow: none; -webkit-box-shadow: none; -moz-box-shadow: none; margin: 0; } nav#access a { font-weight: bold; margin: 11px 28px 0 0; padding: 0; line-height: 21px; } nav#access a:link,nav#access a:visited { text-decoration: none; color: #888; } nav#access a:hover,nav#access li:hover > a,nav#access a:focus,nav#access a:active { text-decoration: none; color: #0063dc; background: none; } #branding .only-search + nav#access div { padding-right: 300px; } nav#access div,nav#access ul { margin: 0; } @media screen and (max-width: 768px) { #branding .only-search + nav#access div { padding-right: 0; } #branding #searchform { display: none; position: static; text-align: center; } #branding .with-image #searchform { max-width: 100%; } #branding .only-search #s,#branding .only-search #s:focus { width: 85%; float: none; } nav#access { margin: 0 0 0 5%; } } @media screen and (max-width: 480px) { nav#access { display: none; } } #content { margin: 0 20% 0 0; width: 80%; } #content .comments-link { display: none; } @media screen and (max-width: 800px) { #main { padding-top: 0; overflow: hidden; } #branding { padding-bottom: 0; } #main #content { margin: 0; } .entry-title,.entry-header .entry-meta { padding-right: 0; } } .singular #content,.left-sidebar.singular #content { margin: 0 10%; width: 80%; } .singular .entry-header,.singular .entry-content,.singular footer.entry-meta,.singular #comments-title { width: 100%; } .hentry,.no-results { border-bottom: 1px dotted #dadada; } .singular .hentry { padding-top: 1em; } .entry-title { padding-bottom: 0; } .entry-title,.entry-title a { font-size: 28px; color: #000; } .entry-title a:hover,.entry-title a:focus,.entry-title a:active { color: #0063dc; background-color: transparent; } @media screen and (max-width: 650px) { .singular .entry-title { padding-top: 0; line-height: 42px; } } .singular .entry-content { margin-top: 0; } #jp-post-flair { margin-top: 3em; } footer.entry-meta .cat-links,footer.entry-meta .sep,footer.entry-meta .tag-links { display: none; } .singular .entry-meta .edit-link a { position: static; } .singular footer.entry-meta { margin-top: 2em; } #comments { display: none; } #site-generator { background: transparent; border-top: none; padding: 0 .5em; } img#wpstats { position: absolute !important; clip: rect(1px 1px 1px 1px); } #secondary { margin-right: 0; width: 16%; text-align: right; } #secondary #s { display: none; } #secondary aside ul { list-style: none; } #secondary .widget-title { color: #999; font-size: 12px; letter-spacing: auto; text-transform: none; } .widget a { font-weight: normal; } .widget a:hover,.widget a:focus,.widget a:active { text-decoration: none; } #everything-after-this-line-is-a-post-level-style---yay-for-wordpress-stripping-comments-from-css { color: #000; } .entry-content p { margin-bottom: 1.3em; } .entry-content p.note { padding: 11px; background-color: #fffdeb; border-bottom: 1px solid #fff9c2; } .entry-content p.warning { padding: 11px; background-color: #fdf8f8; border-bottom: 1px solid #f7dedd; } .entry-content .aside { margin-bottom: 1.3em; padding: 11px; background-color: #f8fdf8; border-bottom: 1px solid #def6df; } .entry-content .aside p:last-child { margin-bottom: 0; } .entry-content h2,.entry-content h3,.entry-content h4 { font-weight: bold; color: #000; text-transform: none; } .entry-content h2 { font-size: 22px; } .entry-content h3 { font-size: 16px; letter-spacing: 0; line-height: 1.3em; margin-bottom: 1em; } .entry-content h4 { font-size: 13px; line-height: 1.4em; margin-bottom: 0; } .entry-content p.flickr-photo,.entry-content p.flickr-photo a,.entry-content p.figure { color: #999; font-size: 14px; } .entry-content p.flickr-photo a:hover,.entry-content p.flickr-photo a:active { color: #0063dc; background-color: transparent; } .entry-content p.flickr-photo .caption { display: block; padding-left: 22px; background: url('http://farm4.staticflickr.com/3329/favicons/72157601614001242_7730.png') no-repeat 0 3px; } .entry-content img { max-width: 100%; height: auto; } .undersized-image-container { text-align: center; } .entry-content code { color: #000; font-family: "Consolas", "Bitstream Vera Sans Mono", "Courier New", Courier, monospace; font-size: 14px; background-color: #F6FcFF; border-bottom: 1px solid #ebf5ff; } .entry-content ul { list-style: disc; } .entry-content ol ol { list-style: lower-alpha; } .entry-content table caption { font-size: 13px; font-weight: bold; color: #000; } .entry-content table tr.odd td,.entry-content table tr.odd th { background-color: #f8f8f8; } .entry-content table td,.entry-content table th { padding: 3px 10px; line-height: 1.4em; } .entry-content table tr th { color: #000; font-weight: bold; text-transform: none; font-size: 13px; letter-spacing: 0; border-top: 1px solid #ddd; } .entry-content table tr td { font-size: 14px; } .entry-content table.data tr td,.entry-content table.data thead tr th { text-align: right; } .entry-content table.tight td,.entry-content table.tight th { padding: 3px 6px; } .entry-content table.tight th { font-size: 12px; } .entry-content table.tight td { font-size: 13px; } @media screen and (max-width: 650px) { .entry-content table td,.entry-content table th { padding: 3px 5px; } .entry-content table tr th { font-size: 12px; } .entry-content table tr td { font-size: 13px; } .entry-content table.tight td,.entry-content table.tight th { padding: 3px 4px; } .entry-content table.tight th { font-size: 11px; } .entry-content table.tight td { font-size: 12px; } } .entry-content blockquote { font-family: inherit; font-style: normal; font-weight: normal; margin: 0 2em 1.3em; padding: 1em; background: #F9F9F9; border-bottom: 1px solid #ececec; } .entry-content blockquote p.source { margin-bottom: 0; } .entry-content .hiring-banner { position: relative; background: #ffebf5; padding: 11px 11px 11px 88px; border-bottom: 1px solid #ffc5e2; margin: 2em 0; } .entry-content .hiring-banner p { margin-bottom: 0; } .entry-content .hiring-banner .group-photo { position: absolute; left: -57px; top: -5px; width: 120px; padding: 6px; background: #fff; -webkit-box-shadow: 1px 1px 8px rgba(50,50,50,0.8); -moz-box-shadow: 1px 1px 8px rgba(50,50,50,0.8); box-shadow: 1px 1px 8px rgba(50,50,50,0.8); -webkit-transform: rotate(-11deg); -moz-transform: rotate(-11deg); -o-transform: rotate(-11deg); -ms-transform: rotate(-11deg); } .entry-content .hiring-banner .group-photo img { display: block; margin: 0; max-width: 100%; } @media screen and (max-width: 800px) { .entry-content .hiring-banner { padding-left: 11px; } .entry-content .hiring-banner .group-photo { display: none; } } #everything-after-this-line-is-for-the-syntaxhighlighter---all-rules-require-important { color: #000; } #main .syntaxhighlighter .lines { border-bottom: 1px solid #ebf5ff !important; } #main .syntaxhighlighter,#main .syntaxhighlighter div,#main .syntaxhighlighter code,#main .syntaxhighlighter table,#main .syntaxhighlighter table td,#main .syntaxhighlighter table tr,#main .syntaxhighlighter table tbody { font-size: 14px !important; } #main .syntaxhighlighter .line.alt1,#main .syntaxhighlighter .line.alt2 { background-color: #fafdff !important; } #main .syntaxhighlighter .line.highlighted { background-color: #fffbd6 !important; } #main .syntaxhighlighter .plain,#main .syntaxhighlighter .plain a { color: #000 !important; } #main .syntaxhighlighter .comments,#main .syntaxhighlighter .comments a { color: #999 !important; } #main .syntaxhighlighter .string,#main .syntaxhighlighter .string a { color: #ff52a9 !important; } #main .syntaxhighlighter .keyword { color: #0034fe !important; font-weight: normal !important; } #main .syntaxhighlighter .preprocessor { color: #417ba9 !important; } #main .syntaxhighlighter .variable { color: #b130c0 !important; } #main .syntaxhighlighter .value { color: #6b77f7 !important; } #main .syntaxhighlighter .functions { color: #002ad5 !important; } #main .syntaxhighlighter .constants { color: #d11e08 !important; } #main .syntaxhighlighter .script { background-color: yellow !important; } #main .syntaxhighlighter .color1,#main .syntaxhighlighter .color1 a { color: #808080 !important; } #main .syntaxhighlighter .color2,#main .syntaxhighlighter .color2 a { color: #ff1493 !important; } #main .syntaxhighlighter .color3,#main .syntaxhighlighter .color3 a { color: red !important; } a[data-flickr-embed] img { width: 800px; } </style> </head> <body class="archive date custom-background wp-embed-responsive jps-theme-flickr-code two-column right-sidebar"> <div class="skip-link"><a class="assistive-text" href="#content">Skip to primary content</a></div><div class="skip-link"><a class="assistive-text" href="#secondary">Skip to secondary content</a></div><div id="page" class="hfeed"> <header id="branding"> <hgroup> <h1 id="site-title"><span><a href="https://code.flickr.net/" rel="home">code.flickr.com</a></span></h1> <h2 id="site-description"></h2> </hgroup> <a href="https://code.flickr.net/"> <img src="https://wp.flickr.net/wp-content/uploads/sites/3/2012/09/code-flickr-com-drawn-header-grey-large.png" width="1000" height="157" alt="code.flickr.com" decoding="async" fetchpriority="high" /> </a> <div class="only-search with-image"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> </form> </div> <nav id="access"> <h3 class="assistive-text">Main menu</h3> <div class="menu-menu-container"><ul id="menu-menu" class="menu"><li id="menu-item-2084" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2084"><a href="http://www.flickr.com/">Flickr</a></li> <li id="menu-item-2085" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2085"><a href="http://blog.flickr.net/">Flickr Blog</a></li> <li id="menu-item-2250" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2250"><a href="http://twitter.com/flickr">@flickr</a></li> <li id="menu-item-2086" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2086"><a href="http://twitter.com/flickrapi">@flickrapi</a></li> <li id="menu-item-2087" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2087"><a href="https://www.flickr.com/services/developer/">Developer Guidelines</a></li> <li id="menu-item-2088" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2088"><a href="http://www.flickr.com/services/api/">API</a></li> <li id="menu-item-2089" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2089"><a href="http://www.flickr.com/jobs/">Jobs</a></li> </ul></div> </nav> </header> <div id="main"> <section id="primary"> <div id="content" role="main"> <header class="page-header"> <h1 class="page-title"> Monthly Archives: <span>September 2015</span> </h1> </header> <article id="post-3181" class="post-3181 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/25/perceptual-image-compression-at-flickr/" rel="bookmark">Perceptual Image Compression at Flickr</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/25/perceptual-image-compression-at-flickr/" title="5:35 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-25T17:35:47-07:00">September 25, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/archieflickr/" title="View all posts by Archie Russell" rel="author">Archie Russell</a></span></span> </div> </header> <div class="entry-content"> <p><i><span style="font-weight:400;">Archie Russell, Peter Norby, Saeideh Bakhshi</span></i></p> <p>At Flickr our users really care about image quality. They <i>also</i> care a lot about how responsive our apps are. Addressing both of these concerns simultaneously is challenging; higher quality images have larger file sizes and are slower to transfer. Slow transfers are especially noticeable on mobile devices. Flickr had historically aimed for high quality at the expense of larger files, but in late 2014 we implemented a method to <em>both</em> maintain image quality and decrease the byte-size of the images we serve to users. As image appearance is very important to our users, we performed an extensive user test before rolling this change out. Here’s how we did it.</p> <h2>Background: JPEG Quality Settings</h2> <div style="width: 355px" class="wp-caption alignright"><img decoding="async" src="https://c1.staticflickr.com/1/755/21476411998_b47d8c2eb6_o.png" alt="" width="345" height="226" /><p class="wp-caption-text">Fig 1. JPEG settings vs file size for a test image.</p></div> <p>JPEG compression has several tuneable knobs. The <b><i>q-value</i></b> is the best known of these; it adjusts the level of spatial detail stored for fine details; a higher q-value typically keeps more detail. However, as q-value gets very close to 100, file size increases dramatically, usually without improving image appearance.</p> <p>If file size and app performance isn’t an issue, dialing up q-value is an easy way to get really nice-looking images; this is what Flickr has done in the past. And if appearance isn’t very important, dialing down q-value is a viable option. But if you want <i>both</i>, you’re kind of stuck. Additionally, q-value isn’t one-size-fits-all, some images look great at q-value 80 while others don’t.</p> <p>Another commonly adjusted setting is chroma-subsampling, which alters the amount of color information stored in a JPEG file. With a setting of 4:4:4, the two chroma (color) channels in a JPG have as much information as the luminance channel. In an image with a setting of 4:2:0, each chroma channel has only a quarter as much information as in an a 4:4:4 image.</p> <table> <tbody> <tr> <td style="text-align:center;"><img decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/573/21660491855_8874629a88_o.jpg" alt="" width="490" height="310" /> q=96, chroma=4:4:4 <em>(125KB)</em></td> <td style="text-align:center;"><img loading="lazy" decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/752/21634483046_65f33afefc_o.jpg" alt="" width="490" height="310" />q=70, chroma=4:4:4 <em>(67KB)</em></td> </tr> <tr> <td style="text-align:center;"><img loading="lazy" decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/781/21660537205_b4bc70dbd5_o.jpg" alt="" width="490" height="310" />q=96, chroma=4:2:0 <em>(62KB)</em></td> <td style="text-align:center;"><img loading="lazy" decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/701/21472657030_c0f4906109_o.jpg" alt="" width="490" height="310" /> q=70, chroma=4:2:0 <em>(62KB)</em></td> </tr> </tbody> </table> <p>Table 1: JPEG stored at different quality and chroma levels. The upper left image is saved at high quality and chroma level; notice the color and detail in the folds of the red flag. The lower right image has the lowest quality; notice artifacts along the right edges of the red flag.</p> <h2>Perceptual JPEG Compression</h2> <p>Ideally we’d have an algorithm which automatically tuned all JPEG parameters to make a file smaller, but which would limit <i>perceptible</i> changes to the image. Technology exists that attempts to do this and can decrease image file size by 30-50%. This compression ratio is highly dependent on image content and dimensions.</p> <table> <tbody> <tr> <td style="text-align:center;"><img loading="lazy" decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/714/21671860802_7c6a2e0c24_o.jpg" alt="" width="490" height="310" /></td> <td style="text-align:center;"><img loading="lazy" decoding="async" class="alignnone" src="https://c1.staticflickr.com/1/667/21495346570_61ce77defd_o.jpg" alt="" width="490" height="310" /></td> </tr> <tr> <td style="text-align:center;">compressed: 112KB</td> <td style="text-align:center;">non-compressed: 224KB</td> </tr> </tbody> </table> <p><span class="caption">Fig 2. <a href="https://c1.staticflickr.com/1/714/21671860802_7c6a2e0c24_o.jpg" target="_blank">Compressed</a> cropped JPEG is 50% smaller than <a href="https://c1.staticflickr.com/1/667/21495346570_61ce77defd_o.jpg" target="_blank">not-compressed</a> cropped JPEG, above, with no obvious defects. Compression ratio is similar for a <a href="https://c1.staticflickr.com/1/751/21642985766_73dc62fc53_o.jpg" target="_blank">compressed 2048-pixel </a>wide JPEG (475KB) of the entire scene and its <a href="https://c1.staticflickr.com/1/623/21481122700_01d36020e7_o.jpg" target="_blank">corresponding not-compressed</a> JPEG (897KB). </span></p> <p>We were pleased with perceptually compressed images in non-structured examinations. The compressed images were smaller and nearly indistinguishable from their sources. But we wanted to really quantify how well the technology worked before considering incorporating it into Flickr. The standard computational tools for evaluating compression, such as SSIM, are fairly simplistic and don’t do a great job at modeling how a user sees things. To really evaluate this technology had to use a better measure of perceptibility: human minds.</p> <p><img decoding="async" src="https://c1.staticflickr.com/1/752/21038236064_32950459f4_o.png" alt="The Gamified Taste Test" width="800" /></p> <p>To test whether our image compression would impact user perception of image quality, we put together a “taste test.” The taste test is constructed as a game with multiple rounds where users look at both compressed and uncompressed images. Users accumulate points the longer they play, and get more points for doing well at the game. We maintained a leaderboard to encourage participation and used only internal testers.The game’s test images came from a diverse collection of 250 images contributed by Flickr staff. The images came from a variety of cameras and included a number of subjects from photographers with varying skill levels.</p> <p><img loading="lazy" decoding="async" src="https://c1.staticflickr.com/1/751/21472835618_213bac1351_o.jpg" alt="sampling of images used in taste test" width="800" height="528" /><br /> <span class="caption">Fig 3. A sampling of images used in our taste test.</span></p> <p>In each round, our test code randomly select a test image, and present two variants of this image side by side. 50% of the time we present the user two identical images; the rest of the time we present one compressed image and one uncompressed image. We ask the tester if the two images look the same or different and we’d expect a user choosing randomly OR a user unable to distinguish the two cases would answer correctly about half the time. We randomly swap the location of the compressed images to compensate for user bias to the left or the right. If testers choose correctly, they are presented with a second question: “Which image did you prefer, and why?”</p> <p><img decoding="async" src="https://c2.staticflickr.com/6/5742/21038190014_37a5a38144_o.png" alt="two kittens in a video game" /><br /> <span class="caption">Fig 4. Screenshot of taste test.</span></p> <p>Our test displays images simultaneously to prevent testers noticing a longer load time for the larger, non-compressed image. The images are presented with either 320, 640, or 1600 pixels on their longest side. The 320 & 640px images are shown for 12 seconds before being dimmed out. The intent behind this detail is to represent how real users interact with our images. The 1600px images stay on screen for 20 seconds, as we expect larger images to be viewed for longer periods of time by real users. We award 100 points per round, regardless of whether a tester chose correctly and also award a bonus of 400 points when a tester correctly identifies whether images were identical or different. We update the tester’s score every five tests so that the user perceives an increasing score without being rewarded immediately for any particular behavior.</p> <h2>Taste Test Outcome and Deployment</h2> <p>We ran our taste test for two weeks and analyzed our results. Although we let users play as long as they liked, we skipped the first result per user as a “warm-up” and considered only the subsequent ten results, this limited the potential for users training themselves to spot compression artifacts. We disregarded users that had fewer than eleven results.</p> <table> <tbody> <tr> <td></td> <td></td> </tr> </tbody> <tbody> <tr> <td style="text-align:center;"><b>images</b></td> <td style="text-align:center;"><b>total results</b></td> <td style="text-align:center;"><b># labeled “identical” by tester</b></td> <td style="text-align:center;"><b>% labeled “identical” by tester</b></td> </tr> <tr> <td>two identical images</td> <td style="text-align:center;"><span style="font-weight:400;">368</span></td> <td style="text-align:center;"><span style="font-weight:400;">253</span></td> <td style="text-align:center;"><span style="font-weight:400;">68.8%</span></td> </tr> <tr> <td>one compressed, one non-compressed</td> <td style="text-align:center;"><span style="font-weight:400;">352</span></td> <td style="text-align:center;"><span style="font-weight:400;">238</span></td> <td style="text-align:center;"><span style="font-weight:400;">67.6%</span></td> </tr> </tbody> </table> <p><span class="caption">Table 2. Taste test results. Testers select “identical” at nearly the same rate, whether the input is identical or not.</span></p> <p>When our testers were presented with two <i>identical</i> images, they thought the images were identical only 68.8% of the time(!), and when presented with a compressed image next to a non-compressed image, our testers thought the images were identical slightly less often: 67.6% of the time. This difference was small enough for us, and our statisticians told us it was statistically insignificant. Our image pairs were so similar that multiple testers thought all images were identical and reported that the test system was buggy. We inspected the images most often labeled different, and found no significant artifacts in the compressed versions.</p> <p>So even in this side-by-side test, perceptual image compression is just barely noticeable when images are presented side-by-side. As the Flickr website wouldn’t ever show compressed and uncompressed images at the same time, and the use of compression had large benefits in storage footprint and site performance, we elected to go forward.</p> <p>At the beginning of 2014 we silently rolled out perceptual-based compression on our image thumbnails (we don’t alter the “original” images uploaded by our users). The slight changes to image appearance went unnoticed by users, but user interactions with Flickr became much faster, especially for users with slow connections, while our storage footprint became much smaller. This was a best-case scenario for us.</p> <p>Evaluating perceptual compression was a considerable task, but it gave the confidence we needed to apply this compression in production to our users. This marked the first time Flickr had adjusted image settings in years, and, it was fun.<br /> <img decoding="async" src="https://c1.staticflickr.com/1/633/21670107841_8b1a09c0a8_o.png" alt="High Score List" width="800" /><br /> <span class="caption">Fig 5. Taste test high score list</span></p> <h2>Epilogue</h2> <p>After eighteen months of perceptual compression at Flickr, we adjusted our settings slightly to shrink images an additional 15%. For our users on mobile devices, 15% fewer bytes per image makes for a much more responsive experience.We had run a taste test on this newer setting and users were were able to spot our compression slightly more often than with our original settings. When presented a pair of identical images, our testers declared these images identical 65.2% of the time, when presented with different images, of our testers declared the images identical 62% of the time. It wasn’t as imperceptible as our original approach, but, we decided it was close enough to roll out.</p> <p>Boy were we wrong! A few very vocal users spotted the compression and didn’t like it at all. The Flickr Help Forum had a very lively thread which <a href="http://petapixel.com/2015/06/08/flickr-decreased-quality-and-increased-compression-and-users-arent-happy/">Petapixel picked up</a>. We <span style="text-decoration:line-through;">beat our heads against the wall</span> considered our options and came up with a middle path between our initial and follow-on approaches, giving us smaller, faster-to-load files while still maintaining the appearance our users expect.</p> <p>Through our use of perceptual compression, combined with our use of <a href="http://code.flickr.net/2015/06/25/real-time-resizing-of-flickr-images-using-gpus/">on-the-fly resize</a> and <a href="http://yahooeng.tumblr.com/post/116391291701/yahoo-cloud-object-store-object-storage-at">COS</a>, we’ve been able to decrease our storage footprint dramatically, while simultaneously improving user experience. It’s a win all around but we’re not done yet — we still have a few tricks up our sleeves.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> <article id="post-3153" class="post-3153 post type-post status-publish format-standard hentry category-hadoop category-infrastructure category-kittens tag-hadoop tag-hbase tag-lambda-architecture tag-magic-view tag-pig-latin tag-storm"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/03/powering-flickrs-magic-view/" rel="bookmark">Powering Flickr’s Magic view by fusing bulk and real-time compute</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/03/powering-flickrs-magic-view/" title="5:30 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-03T17:30:21-07:00">September 3, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/bhautikj/" title="View all posts by Bhautik Joshi" rel="author">Bhautik Joshi</a></span></span> </div> </header> <div class="entry-content"> <blockquote> <h1>Try it for yourself!</h1> <p>You can try out Flickr’s Magic View on your own photos <a href="https://www.flickr.com/cameraroll">here</a>, and you can download a working code sample of the simplified lambda architecture here: <a href="https://github.com/yahoo/simplified-lambda">https://github.com/yahoo/simplified-lambda</a></p></blockquote> <h1>Introduction</h1> <p><span style="font-weight:400;">In this post we’re going to talk about how we came up with a novel revision of the Lambda Architecture for fusing large-scale bulk compute with streaming compute to power Flickr’s Magic View. We were able to create a responsive, real time database operating at a scale of tens of billions of records, with tens to hundreds of millions of records updated per day.</span> We turned to <a href="http://yahoohadoop.tumblr.com/">Yahoo’s Hadoop stack</a> to find a way to build this at the massive scale we needed.</p> <p><img loading="lazy" decoding="async" class="aligncenter size-medium wp-image-3156" style="text-align:center;" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?w=800" alt="Magic View" width="800" height="593" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg 1600w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=150,111 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=800,593 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=768,569 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=1024,758 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=1536,1138 1536w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=405,300 405w" sizes="auto, (max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 1. Magic View in action</em></strong></p> <h2>Motivation: the Magic View</h2> <p><span style="font-weight:400;">Flickr’s Magic View takes the hassle out of organizing your photos by applying our computer-vision technology to automatically recognize objects or styles in your photos and present them to you in the Camera Roll’s scrolling view. This all happens in real time </span><span style="font-weight:400;">—</span><span style="font-weight:400;"> as soon as a photo is uploaded, it is categorized and placed into the Magic View.</span></p> <h2>Aggregating computer vision tags</h2> <p><span style="font-weight:400;">When a photo is uploaded, it is processed by a computer vision pipeline to generate a set of computer vision</span><i><span style="font-weight:400;"> tags</span></i><span style="font-weight:400;">, which are text labels of the contents of the image. We already had an existing architecture for stream computation of tags on upload, but to implement the Magic View, we needed to maintain per-user reverse indexes and some aggregations of the tags. And we needed to make sure all the data was consistent </span><span style="font-weight:400;">—</span><span style="font-weight:400;"> if a photo was added, removed or updated these indexes and aggregations would have to be updated to reflect this. Finally, we needed to initialize the system with tags for 12 billion photos and videos and run periodic backfills (every time we improved our computer vision algorithms and to cover cases where the stream compute missed images).</span></p> <h2>The Problem</h2> <p><span style="font-weight:400;">We initially computed a snapshot of the Magic View indexes and aggregations using map-reduce (via </span><a href="http://oozie.apache.org/"><span style="font-weight:400;">Apache Oozie</span></a><span style="font-weight:400;"> and </span><a href="http://pig.apache.org/"><span style="font-weight:400;">Apache Pig</span></a><span style="font-weight:400;">), and we were happy with the quick turnaround time (about 7 hours). We considered updating Magic View as a daily batch job, but soon realized this would not give our users the responsive, “live” experience we wanted. So, we built a streaming data layer using </span><a href="https://storm.apache.org/"><span style="font-weight:400;">Apache Storm</span></a><span style="font-weight:400;"> and were soon able to update the categories in Magic View in real-time.</span></p> <p><span style="font-weight:400;">The next time we needed to run a backfill, we explored using this streaming layer to load the data. Unfortunately, the overhead of the read-modify-write process was simply too much for a load of this size — after kicking off the process we estimated it would take <i><span style="font-weight:400;">28 days</span></i><span style="font-weight:400;"> this way — much longer than the seven hours we had achieved with a bulk load.</span></span></p> <p><span style="font-weight:400;">Twenty-eight days was a non-starter – we realized we needed a way to update our bulk aggregations independently of the real-time data streaming in. Solving this problem is how we arrived at our revision to Lambda Architecture. Before digging into the solution, let’s do a quick review of the Lambda Architecture. If you’re already familiar with it, you can skip this next section.</span></p> <h2>The Lambda Architecture</h2> <p>We’ll start with Nathan Marz’s book ‘<a href="https://www.manning.com/books/big-data">Big Data</a>’, which proposes the database concept of ‘Lambda Architecture.’ In his analysis, he states that a database query can be represented as a function – Query – which operates on all the data:</p> <pre style="text-align:center;">result = Query(all data)</pre> <p><span style="font-weight:400;">In the Lambda architecture, a traditional database is replaced with both a </span><i><span style="font-weight:400;">real time</span></i><span style="font-weight:400;"> and a </span><i><span style="font-weight:400;">bulk</span></i><span style="font-weight:400;"> database. Then query function becomes a “combiner” function of independent queries to each database:</span></p> <pre style="text-align:center;">result = Combiner(Query(real time data) + Query(bulk data))</pre> <p><span style="font-weight:400;">An example of a typical Lambda Architecture is shown in figure 2. It is powered by an append-only queue for its system of record, which is fed by a real time stream of events. Periodically, all the data in the queue is fed into a bulk computation which pre-processes the data to optimize it for queries, and stores these aggregations in a bulk compute database. The real time event stream drives a stream computer, which processes the incoming events into real time aggregations. A query then goes via a query combiner, which queries both the bulk and real time databases, computes the combination, and stores the result.</span></p> <p><img loading="lazy" decoding="async" class="aligncenter size-medium wp-image-3157" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?w=800" alt="Typical Lambda Architecture" width="800" height="339" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png 1039w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=150,64 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=800,339 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=768,325 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=1024,434 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=500,212 500w" sizes="auto, (max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 2. Typical Lambda Architecture</em></strong></p> <p><span style="font-weight:400;">While relatively new, Lambda Architecture has enjoyed popularity and a number of concrete implementations have been built. Some significant examples are the distributed analytics platform </span><a href="http://druid.io/"><span style="font-weight:400;">druid</span></a><span style="font-weight:400;">, </span><a href="https://github.com/twitter/summingbird"><span style="font-weight:400;">Twitter’s Summingbird</span></a><span style="font-weight:400;">, and </span><a href="https://github.com/velvia/FiloDB"><span style="font-weight:400;">FiloDB</span></a><span style="font-weight:400;">. These implementations conveniently abstract away the databases behind the query combiner. </span></p> <p><span style="font-weight:400;">A significant advantage with this style of architecture is robustness and fault-tolerance via eventual consistency. If a piece of data is skipped in the real time compute there is a guarantee that it will eventually appear in the bulk compute database.</span></p> <p><span style="font-weight:400;">Criticism of the Lambda Architecture has centred around the </span><a href="http://radar.oreilly.com/2014/07/questioning-the-Lambda-architecture.html"><span style="font-weight:400;">complicated nature of the combiner</span></a><span style="font-weight:400;">. The combiner incurs a developer and systems cost from the need to maintain two different databases. It can be challenging to make sure both systems give the same result. Merging the two queries can become complicated, and finally, more points of failure may be introduced.</span></p> <h2>The “Ah-ha” Moment</h2> <p><span style="font-weight:400;">Back to the problem. The data access layer we used for streaming compute uses the atomic read-modify-write pattern to ensure we write consistent data, one record-at-a-time to </span><a href="http://hbase.apache.org/"><span style="font-weight:400;">Apache HBase</span></a><span style="font-weight:400;"> (a BigTable-style, non-relational database). Again, since this pattern was so much slower in the backfill case we needed to figure out how to get both consistent updates for streaming </span><i><span style="font-weight:400;">and </span></i><span style="font-weight:400;"> fast loads of the full dataset. Since our bulk data was static, we realized that if we relaxed the consistency constraint we could just run a fast, streaming, write-only load of the bulk data, bringing the load time back down to hours instead of days.</span></p> <p><span style="font-weight:400;">But how could we get around the consistency requirements? We didn’t want a bulk load to clobber data being written from the real time compute process. The insight was that we could just write bulk and streaming data to different column families in the same HBase row. So we added the concept of </span><i><span style="font-weight:400;">real time columns</span></i><span style="font-weight:400;"> and </span><i><span style="font-weight:400;">bulk columns</span></i><span style="font-weight:400;"> in a single row. Basically, bulk loads write to one set of columns and real time writes go to a different set of columns. Since HBase columns are sparse and data is updated relatively slowly we don’t pay much in storage or IO.</span></p> <p><span style="font-weight:400;">We could now simplify the equation back to:</span></p> <pre style="text-align:center;">result = Combiner(Query(data))</pre> <p><span style="font-weight:400;">The two sets of columns are managed separately by the real time and bulk subsystems. At query time, we perform a single fetch using the HBase API to get </span><i><span style="font-weight:400;">both</span></i><span style="font-weight:400;"> the bulk and real time data. A separate combiner process assembles the final result. </span></p> <h1>Implementation</h1> <p><img loading="lazy" decoding="async" class="aligncenter size-medium wp-image-3159" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?w=800" alt=" Magic View backend system overview" width="800" height="347" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png 1133w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=150,65 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=800,347 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=768,333 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=1024,444 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=500,217 500w" sizes="auto, (max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 3. Magic View Architecture</em></strong></p> <p><span style="font-weight:400;">Figure 3 shows an overview of the system and our enhanced Lambda architecture. For the purposes of this discussion, a convenient abstraction is to consider that each row in the HBase table represents the current state of a given photo. The combiner stage is abstracted into a single Java process, which collects data from HBase and runs transformations on the data and sends it to a </span><a href="http://redis.io/"><span style="font-weight:400;">Redis</span></a><span style="font-weight:400;"> cache which is used by the serving layer for the site.</span></p> <h2>Consistency on read in HBase — the combiner</h2> <p><span style="font-weight:400;">We have two sets of columns to go with each row in HBase: bulk and real time. The combiner determines the final value for each attribute at read. In the case where data exists for real time but not for bulk (or vice versa) then there is only one value to choose. In the case where they both exist we always choose the real time value. This keeps the combiner very simple and fast. </span></p> <p><span style="font-weight:400;">There is a trick though – whenever we do a backfill, we may need to repair the row since the backfill data may be newer than any real time data that is already present. It turns out this slows down the backfill from seven hours to about 14 — still far faster than loading with read-modify-write.</span></p> <h2>Production throughput</h2> <p><span style="font-weight:400;">At scale, this architecture has been able to keep up very comfortably with production load. We can simultaneously run backfills to HBase and serve user information at the same time without impacting latency or the user experience.</span></p> <h2>User experience</h2> <p><span style="font-weight:400;">An important measure for how the system works is how the viewer perceives it. The slowest part of the system is paging data from HBase into the serving cache; median time for above-the-fold latency – i.e. enough data is available to render the page – is around 10ms. </span></p> <h2>Future directions</h2> <p><span style="font-weight:400;">Our experience has been very positive so far with Magic View and we’re looking at how we might enable users to browse their photos in other dimensions (location or color for example). Early tests have shown that building an OLAP or data cube in this architecture is certainly possible but it’s less clear that it will scale well.</span></p> <p><i><span style="font-weight:400;">Contributors: Peter Welch, Bhautik Joshi, Hugo Haas, Srinivasan Singanallur, Ayan Ray, Pierre Garrigues, Ben Firestone, Sai Madhavan, Tim Miller</span></i></p> <p><em>Thanks to Nathan Marz for reviewing this post.</em></p> <div class="hiring-banner"> <p class="group-photo"><a title="Flickr September 2014 by Bhautik Joshi, on Flickr" href="https://www.flickr.com/photos/captin_nod/14965686478/"><img loading="lazy" decoding="async" src="https://farm4.staticflickr.com/3893/14965686478_9278dbe39c_m.jpg" alt="Flickr September 2014" width="120" height="80" /></a></p> <p>Like this post? Have a love of online photography? Want to work with us? Flickr is hiring <strong>mobile, back-end and front-end engineers</strong>, in our San Francisco office. <strong>Find out more at <a href="https://www.flickr.com/jobs/">flickr.com/jobs</a></strong>.</p> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/hadoop/" rel="category tag">hadoop</a>, <a href="https://code.flickr.net/category/infrastructure/" rel="category tag">infrastructure</a>, <a href="https://code.flickr.net/category/kittens/" rel="category tag">kittens</a> </span> <span class="sep"> | </span> <span class="tag-links"> <span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="https://code.flickr.net/tag/hadoop/" rel="tag">hadoop</a>, <a href="https://code.flickr.net/tag/hbase/" rel="tag">hbase</a>, <a href="https://code.flickr.net/tag/lambda-architecture/" rel="tag">lambda architecture</a>, <a href="https://code.flickr.net/tag/magic-view/" rel="tag">magic view</a>, <a href="https://code.flickr.net/tag/pig-latin/" rel="tag">pig latin</a>, <a href="https://code.flickr.net/tag/storm/" rel="tag">storm</a> </span> </footer> </article> <article id="post-3114" class="post-3114 post type-post status-publish format-standard hentry category-uncategorized tag-api tag-caching tag-data-model tag-eviction tag-model"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/01/the-data-freshener/" rel="bookmark">The Data Freshener</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/01/the-data-freshener/" title="4:26 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-01T16:26:49-07:00">September 1, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/ericsoco/" title="View all posts by ericsoco" rel="author">ericsoco</a></span></span> </div> </header> <div class="entry-content"> <div style="width:340px;"> <a href="https://www.flickr.com/photos/hellomokona/956099245/">https://www.flickr.com/photos/hellomokona/956099245/</a><br /> <em>So fresh</em> </div> <p> </p> <h2>Change</h2> <p>You may have noticed some changes in Flickr a couple months back. Like, half the site changed. 95% even, by some metrics. Some say CHANGE IT BACK! while others welcome change. Whatever your thoughts, the changes are here, and they mean things. For example, they mean new visual design and better usability. They mean a faster site. Unfortunately, up until recently, they also meant more stale data. Yuck.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/spcbrass/4388396268/" title="Change by spcbrass, on Flickr"><img loading="lazy" decoding="async" src="https://farm3.staticflickr.com/2759/4388396268_87224c6556_z.jpg" width="640" height="480" alt="Change"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Change</em><br />   </div> <p>Why? What? Well…here’s the deal. We have a new-ish frontend stack we’ve been using for the past couple years now. It’s an <a href="http://isomorphic.net/javascript">isomorphic</a> <a href="https://en.wikipedia.org/wiki/Single-page_application">single-page application</a>, runs on <a href="http://nodejs.org">node.js</a>, and is generally awesome. We call it Reboot.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/wafer/6897507098/" title="hi there / i am the computer by waferbaby, on Flickr"><img loading="lazy" decoding="async" src="https://farm6.staticflickr.com/5466/6897507098_78d4c5a056_z.jpg" width="640" height="425" alt="hi there / i am the computer"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Reboot</em><br />   </div> <p>In the World of Reboot, we treat data with kid gloves. We <3 data. We never want to give it up, never want to let it down. Once we pull data from our APIs, we store the fetched data in your browser so that we don’t have to fetch it again the next time it’s needed. This means faster page loads and faster navigation, and less API traffic (and thus a more stable and scalable API). The data cached in your browser exists as long as the current Reboot session — until you refresh or leave Reboot for a non-Rebooted page.</p> <p>However, this also meant that data could become stale. You change the date taken of your photo, someone else adds a comment, you navigate to a page with cached data…and you don’t see the changes. Wat? Yeah. So, this was not a huge problem until we moved lots of pages onto Reboot in the beginning of May. From that point forward, most Flickr user sessions have spent their entirety on Reboot, feeding off the same stale loaves of cached data.</p> <div style="width:480px;"> <a href="https://www.flickr.com/photos/recyclethis/157108084/">https://www.flickr.com/photos/recyclethis/157108084/</a><br /> <em>Staleness</em><br />   </div> <h2>The thinking (design / prototypes)</h2> <p>We considered a number of possibilities for freshening up data during a user session. A brief history of the strategies we sampled, and their results:</p> <h3>1. Refresh on update</h3> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/the_family_farm/2571858493/" title="Ice Tea by MzScarlett / A.K.A. Michelle, on Flickr"><img loading="lazy" decoding="async" src="https://farm4.staticflickr.com/3075/2571858493_34a1f75ac3_z.jpg" width="640" height="480" alt="Ice Tea"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>The first stab focused on updating data locally after it was changed by the user. Most of our simpler use cases already updated as expected, but some trickier cases with indirect relationships did not. For example, changing the date taken of a photo updated the data model for the photo, but deleting a photo did not necessarily ensure the photo was removed from all the cached albums, groups, and galleries to which it belonged. (Note that the photo was removed correctly from the backend, just not from the cached representation of those entities on the client.)</p> <p>Cleaning up these relationships using change events between models helped, but didn’t solve all our problems. When someone outside of the local session (read: another user) changed data, it would not reflect in the current session. The only way to catch changes from outside the current session was to be more aggressive about evicting models.</p> <h3>2. Nuclear option</h3> <div style="width:480px;"> <a href="https://www.flickr.com/photos/sdasmarchives/8091816484/">https://www.flickr.com/photos/sdasmarchives/8091816484/</a><br />   </div> <p>The pendulum swung all the way in the other direction — instead of surgical removal of data models we knew to be out-of-date, what would happen if we removed all cached data on every navigation? This prototype was quick to build, and incredibly destructive. By doing this, all our cached data always remained as fresh as could be, but we essentially reverted to Web 1.0 — with the exception of the Reboot framework, everything was reloaded on every page.</p> <p>Not surprisingly, this blew up API traffic (locally only! did not unleash that disaster at scale), and inflated page load times like a Jeff Koons sculpture. It did give us some baseline timing metrics we could point to as worst-case scenarios, however. The next step was to swing the pendulum back toward the middle — to a carefully-knitted solution that would preserve fast page loads and navigation, while ensuring the freshest data we could serve up.</p> <h3>3. Refetch on navigate</h3> <div style="width:340px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/daveemerson/4296539211/" title="fetched by Dave's Domain, on Flickr"><img loading="lazy" decoding="async" src="https://farm5.staticflickr.com/4029/4296539211_c054e6b762_z.jpg" width="498" height="640" alt="fetched"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>At this point, our challenge was to find a solution that would keep navigation fast, API traffic slim, and pick up all changes to session data, whether local or remote. We ended up with a solution we call “refetching”: evicting and requesting new data models as the model is needed by the application. But when?<br /> We could refetch periodically or on a user action; we determined that the best time to trigger a refetch was on navigation — when the user navigates, cached models become eligible for refetching. Specifically, when the user navigates between sections of the site, refetching is triggered. This proved to be the happiest medium between speed and freshness.</p> <p>A high-level outline of how the refetching strategy works:</p> <ul> <li>The user loads a page; data are requested from the API, and models are cached. As new models are created, they’re marked as being fresh.</li> <li>The user navigates to another site section (e.g. Photostream → Search); all freshness marks are removed from all models. They’re now all eligible for refetching.</li> <li>As Reboot builds the new page, it requests data models from the cache. Since they no longer have their seal of freshness, they are refetched, and marked as fresh once retrieved and cached.</li> </ul> <p>One important note — refetching is not triggered on browser back/forward navigation. Users expect near-immediate navigation, thanks to browser caching, when navigating to already-viewed content. Therefore, we refetch only when the user clicks a link to navigate to a new site section.</p> <h3>4. Miscellany</h3> <p>There were a couple other options we considered and rejected from the start, but they’re worth mentioning here.</p> <p>One was a <a href="https://en.wikipedia.org/wiki/Time_to_live">TTL (time-to-live) algorithm</a>, commonly used in caching applications. TTL algorithms expire data and evict from the cache a certain amount of time after they’re written or last updated. The arbitrary nature of TTL would mean that users would sometimes have fresh data and sometimes stale; it would be fresh more often than without any solution, but freshness would vary arbitrarily and would not result in much of an improvement on user experience.</p> <p>The other was to write an algorithm that tracks the amount of time since a data model was last accessed, and refetch when it grows too old. While this sounded interesting at first, it has the same flaw as a standard TTL algorithm — freshness becomes arbitrary. It’s also more complex to implement, and might end up not being worth the complexity.</p> <h2>The doing (implementation)</h2> <p>So that was it! Refetch on navigate, all done. Right?….of course not. With the general strategy in place, the devil started sneaking around in all the details. Some of the highlights:</p> <h3>Exemptions</h3> <p>It proved to be not the best idea to evict on all navigation. For example, in Reboot we often preload photo metadata models on pages with lists of photos, in order to make navigation into the photo page snappy. The refetch setup therefore has an exemption config that allows us to easily retain models when navigating into, away from, or between specific site sections.</p> <h3>Child models</h3> <p>We often have parent-child associations between data models. For example, the data model for a photo has a reference to a data model for the author of the photo. When the photo model is refetched, the person model must be refetched as well. This means the function doing the eviction and refetching has to recurse through all child models.</p> <h3>Collections</h3> <p>An issue similar to child models above, but more complex, is the case of a model containing a list of other models. For example, the data model for a person’s photostream contains a list of photo models.</p> <p>What made this particularly tricky is pagination and filtering — say you load the first 2 pages of your photostream, set your view filter to private, jump to page 5, switch the view to “Date taken”, and navigate away and back to your photostream…imagine the mess of different models with partially-loaded collections. Evicting one parent model, and its children, might evict photo models from the collection within another, without properly refetching. The solution here actually lay in the controller responsible for fetching pages: if a requested page of models is not already completely in-cache, a refetch will always happen to ensure we have all the data, in its freshest state.</p> <h3>Refetch only once per page view</h3> <p>Critical to the refetch-on-navigation strategy is to refetch only once per navigation. This was not too difficult, but essential to get right. We accomplish this by adding a flag when a model is initially fetched and upserted into the cache. When navigating to a new, non-exempt site section, all those flags are cleared, and any model requested by the new page will be refetched. When refetched, the model is again upserted into the cache and marked as fresh, until the next navigation.</p> <h2>But did it fresh?</h2> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/libellule24/4438085129/" title="Go on without me by Senorita Lena, on Flickr"><img loading="lazy" decoding="async" src="https://farm5.staticflickr.com/4019/4438085129_58cbbeb2f9_z.jpg" width="640" height="427" alt="Go on without me"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>With the thinking and the doing out of the way, it was time to push all this to production. Because these changes are essentially pulling the rug out from underneath the data layer on every navigation, we had to tread very carefully in order to prevent any negative impact to the end user experience.</p> <p>We did very thorough manual and automated testing across all of Reboot. We left the feature turned on for staff users for a while, to be able to respond to any bug reports. Finally, the time came to test on Real People. There were three things we needed to keep an eye on: errors (of course), impact on page navigation timing, and API traffic. Since refetching implies more requests for data, we needed to be sure that we were keeping the user experience smooth and fast, and also that we weren’t blowing up our data centers.</p> <div style="width:480px;">  <br /> <a href="https://www.flickr.com/photos/karolfranks/6296290871/">https://www.flickr.com/photos/karolfranks/6296290871/</a><br /> <em>All in</em><br />   </div> <p>In order to get a good read on these things, though, we had to go all in. Letting in just a small percentage of users would not give reliable numbers for timing or traffic impacts, due to the noise inherent in relatively small sample sizes. So, we did something unusual: we turned on refetching for all users for a short period of time. We flipped on refetching and kept an eagle eye on our stats for 2 hours, then reverted; then, we took a careful look at the aggregated data to see how the experiment went.</p> <p>Surprisingly, the impact on both timing and traffic was relatively low. After some thought, we decided this is most likely because the changes disproportionately impact people on long sessions, say a Flickr tab open for hours or days. Most people don’t hang around that long; they come, they go. Also, the photo page represents north of 90% of our page views, and is exempt from refetching (see Exemptions above).</p> <p>So where did we end up? A negligible bump in navigation timing and API traffic, and fresher data for all. Perhaps an anticlimactic resolution, but the story we’ve heard today outlines a serious consideration for anyone building an application with a data caching layer: keep in mind from the beginning how you plan to deal with stale data, but in a way that keeps all the other benefits of a single-page application.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/pinguino/6800265589/" title="#CCC is a breadcat by pinguino, on Flickr"><img loading="lazy" decoding="async" src="https://farm8.staticflickr.com/7151/6800265589_3204a4723b_z.jpg" width="640" height="427" alt="#CCC is a breadcat"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Busting through staleness. Yep.</em> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> <span class="sep"> | </span> <span class="tag-links"> <span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="https://code.flickr.net/tag/api/" rel="tag">api</a>, <a href="https://code.flickr.net/tag/caching/" rel="tag">caching</a>, <a href="https://code.flickr.net/tag/data-model/" rel="tag">data model</a>, <a href="https://code.flickr.net/tag/eviction/" rel="tag">eviction</a>, <a href="https://code.flickr.net/tag/model/" rel="tag">model</a> </span> </footer> </article> </div> </section> <div id="secondary" class="widget-area" role="complementary"> <aside id="jetpack-search-filters-3" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-3-wrapper" class="jetpack-instant-search-wrapper"> <div class="jetpack-search-form"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> <input type="hidden" name="orderby" value="" /><input type="hidden" name="order" value="" /></form> </div> <h4 class="jetpack-search-filters-widget__sub-heading"> Categories </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uncategorized"> Uncategorized (136) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="geo"> geo (12) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="kittens"> kittens (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="change-log"> changelog (7) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uploadr"> uploadr (6) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Tags </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="api"> api (24) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="geo"> geo (13) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="machine-tags"> machine tags (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="javascript"> javascript (9) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="kittentuesday"> kittentuesday (8) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Year </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2022-01-01 00:00:00" > 2022 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2021-01-01 00:00:00" > 2021 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2018-01-01 00:00:00" > 2018 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2017-01-01 00:00:00" > 2017 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2016-01-01 00:00:00" > 2016 (5) </a> </li> </ul> </div></aside> <aside id="recent-posts-2" class="widget widget_recent_entries"> <h3 class="widget-title">Recent Posts</h3> <ul> <li> <a href="https://code.flickr.net/2022/02/14/safer-internet-day-and-open-source-codes-of-conduct/">Safer Internet Day and Open Source Codes of Conduct</a> </li> <li> <a href="https://code.flickr.net/2022/01/04/a-pluggable-solution-for-api-observability-on-our-php-system/">A Pluggable Solution for API Observability on our PHP System</a> </li> <li> <a href="https://code.flickr.net/2021/11/22/flickr-engineering-team-vision-guiding-principles/">Flickr Engineering Team Vision & Guiding Principles</a> </li> <li> <a href="https://code.flickr.net/2018/04/20/together/">Together</a> </li> <li> <a href="https://code.flickr.net/2017/03/07/introducing-similarity-search-at-flickr/">Introducing Similarity Search at Flickr</a> </li> </ul> </aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3> <ul> <li><a href='https://code.flickr.net/2022/02/'>February 2022</a></li> <li><a href='https://code.flickr.net/2022/01/'>January 2022</a></li> <li><a href='https://code.flickr.net/2021/11/'>November 2021</a></li> <li><a href='https://code.flickr.net/2018/04/'>April 2018</a></li> <li><a href='https://code.flickr.net/2017/03/'>March 2017</a></li> <li><a href='https://code.flickr.net/2017/01/'>January 2017</a></li> <li><a href='https://code.flickr.net/2016/09/'>September 2016</a></li> <li><a href='https://code.flickr.net/2016/05/'>May 2016</a></li> <li><a href='https://code.flickr.net/2016/04/'>April 2016</a></li> <li><a href='https://code.flickr.net/2016/03/'>March 2016</a></li> <li><a href='https://code.flickr.net/2015/12/'>December 2015</a></li> <li><a href='https://code.flickr.net/2015/11/'>November 2015</a></li> <li><a href='https://code.flickr.net/2015/09/' aria-current="page">September 2015</a></li> <li><a href='https://code.flickr.net/2015/07/'>July 2015</a></li> <li><a href='https://code.flickr.net/2015/06/'>June 2015</a></li> <li><a href='https://code.flickr.net/2015/03/'>March 2015</a></li> <li><a href='https://code.flickr.net/2014/10/'>October 2014</a></li> <li><a href='https://code.flickr.net/2014/08/'>August 2014</a></li> <li><a href='https://code.flickr.net/2014/07/'>July 2014</a></li> <li><a href='https://code.flickr.net/2014/05/'>May 2014</a></li> <li><a href='https://code.flickr.net/2014/04/'>April 2014</a></li> <li><a href='https://code.flickr.net/2014/02/'>February 2014</a></li> <li><a href='https://code.flickr.net/2013/09/'>September 2013</a></li> <li><a href='https://code.flickr.net/2013/06/'>June 2013</a></li> <li><a href='https://code.flickr.net/2013/03/'>March 2013</a></li> <li><a href='https://code.flickr.net/2012/12/'>December 2012</a></li> <li><a href='https://code.flickr.net/2012/10/'>October 2012</a></li> <li><a href='https://code.flickr.net/2012/07/'>July 2012</a></li> <li><a href='https://code.flickr.net/2012/06/'>June 2012</a></li> <li><a href='https://code.flickr.net/2012/05/'>May 2012</a></li> <li><a href='https://code.flickr.net/2012/04/'>April 2012</a></li> <li><a href='https://code.flickr.net/2012/02/'>February 2012</a></li> <li><a href='https://code.flickr.net/2012/01/'>January 2012</a></li> <li><a href='https://code.flickr.net/2011/12/'>December 2011</a></li> <li><a href='https://code.flickr.net/2011/10/'>October 2011</a></li> <li><a href='https://code.flickr.net/2011/09/'>September 2011</a></li> <li><a href='https://code.flickr.net/2011/08/'>August 2011</a></li> <li><a href='https://code.flickr.net/2011/07/'>July 2011</a></li> <li><a href='https://code.flickr.net/2011/06/'>June 2011</a></li> <li><a href='https://code.flickr.net/2011/03/'>March 2011</a></li> <li><a href='https://code.flickr.net/2011/02/'>February 2011</a></li> <li><a href='https://code.flickr.net/2011/01/'>January 2011</a></li> <li><a href='https://code.flickr.net/2010/11/'>November 2010</a></li> <li><a href='https://code.flickr.net/2010/10/'>October 2010</a></li> <li><a href='https://code.flickr.net/2010/09/'>September 2010</a></li> <li><a href='https://code.flickr.net/2010/08/'>August 2010</a></li> <li><a href='https://code.flickr.net/2010/07/'>July 2010</a></li> <li><a href='https://code.flickr.net/2010/05/'>May 2010</a></li> <li><a href='https://code.flickr.net/2010/04/'>April 2010</a></li> <li><a href='https://code.flickr.net/2010/03/'>March 2010</a></li> <li><a href='https://code.flickr.net/2010/02/'>February 2010</a></li> <li><a href='https://code.flickr.net/2010/01/'>January 2010</a></li> <li><a href='https://code.flickr.net/2009/12/'>December 2009</a></li> <li><a href='https://code.flickr.net/2009/11/'>November 2009</a></li> <li><a href='https://code.flickr.net/2009/10/'>October 2009</a></li> <li><a href='https://code.flickr.net/2009/09/'>September 2009</a></li> <li><a href='https://code.flickr.net/2009/07/'>July 2009</a></li> <li><a href='https://code.flickr.net/2009/06/'>June 2009</a></li> <li><a href='https://code.flickr.net/2009/05/'>May 2009</a></li> <li><a href='https://code.flickr.net/2009/04/'>April 2009</a></li> <li><a href='https://code.flickr.net/2009/03/'>March 2009</a></li> <li><a href='https://code.flickr.net/2009/02/'>February 2009</a></li> <li><a href='https://code.flickr.net/2009/01/'>January 2009</a></li> <li><a href='https://code.flickr.net/2008/12/'>December 2008</a></li> <li><a href='https://code.flickr.net/2008/11/'>November 2008</a></li> <li><a href='https://code.flickr.net/2008/10/'>October 2008</a></li> <li><a href='https://code.flickr.net/2008/09/'>September 2008</a></li> <li><a href='https://code.flickr.net/2008/08/'>August 2008</a></li> <li><a href='https://code.flickr.net/2008/07/'>July 2008</a></li> <li><a href='https://code.flickr.net/2008/06/'>June 2008</a></li> <li><a href='https://code.flickr.net/2008/05/'>May 2008</a></li> <li><a href='https://code.flickr.net/2008/04/'>April 2008</a></li> </ul> </aside><aside id="categories-2" class="widget widget_categories"><h3 class="widget-title">Categories</h3> <ul> <li class="cat-item cat-item-11749740"><a href="https://code.flickr.net/category/api-2/">API</a> </li> <li class="cat-item cat-item-564792"><a href="https://code.flickr.net/category/change-log/">changelog</a> </li> <li class="cat-item cat-item-5784"><a href="https://code.flickr.net/category/event/">event</a> </li> <li class="cat-item cat-item-29160"><a href="https://code.flickr.net/category/geo/">geo</a> </li> <li class="cat-item cat-item-139037766"><a href="https://code.flickr.net/category/hadoop/">hadoop</a> </li> <li class="cat-item cat-item-32"><a href="https://code.flickr.net/category/infrastructure/">infrastructure</a> </li> <li class="cat-item cat-item-139037765"><a href="https://code.flickr.net/category/kittens/">kittens</a> </li> <li class="cat-item cat-item-20156"><a href="https://code.flickr.net/category/labs/">labs</a> </li> <li class="cat-item cat-item-171"><a href="https://code.flickr.net/category/meta/">meta</a> </li> <li class="cat-item cat-item-7092"><a href="https://code.flickr.net/category/metrics/">metrics</a> </li> <li class="cat-item cat-item-139037764"><a href="https://code.flickr.net/category/open-source/">open source</a> </li> <li class="cat-item cat-item-1930"><a href="https://code.flickr.net/category/performance/">performance</a> </li> <li class="cat-item cat-item-304"><a href="https://code.flickr.net/category/photos/">photos</a> </li> <li class="cat-item cat-item-2373"><a href="https://code.flickr.net/category/search/">search</a> </li> <li class="cat-item cat-item-1"><a href="https://code.flickr.net/category/uncategorized/">Uncategorized</a> </li> <li class="cat-item cat-item-249276"><a href="https://code.flickr.net/category/uploadr/">uploadr</a> </li> <li class="cat-item cat-item-412"><a href="https://code.flickr.net/category/video/">video</a> </li> <li class="cat-item cat-item-830560"><a href="https://code.flickr.net/category/xulrunner/">xulrunner</a> </li> </ul> </aside><aside id="meta-2" class="widget widget_meta"><h3 class="widget-title">Meta</h3> <ul> <li><a href="https://code.flickr.net/wp-login.php">Log in</a></li> <li><a href="https://code.flickr.net/feed/">Entries feed</a></li> <li><a href="https://code.flickr.net/comments/feed/">Comments feed</a></li> <li><a href="https://wordpress.org/">WordPress.org</a></li> </ul> </aside> </div> </div> <footer id="colophon" role="contentinfo"> <div id="site-generator"> © 2025 Flickr, Inc. All rights reserved. | Powered by <a href="https://wpvip.com/?utm_source=vip_powered_wpcom&utm_medium=web&utm_campaign=VIP%20Footer%20Credit&utm_term=code.flickr.net" rel="generator nofollow" class="powered-by-wpcom">WordPress VIP</a> </div> </footer> </div> <div class="jetpack-instant-search__widget-area" style="display: none"> <div id="jetpack-search-filters-2" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-2-wrapper" class="jetpack-instant-search-wrapper"> </div></div> </div> <script type="text/javascript" src="https://code.flickr.net/wp-includes/js/dist/hooks.min.js?m=1742994401g" ></script><script type="text/javascript" src="https://code.flickr.net/wp-includes/js/dist/i18n.min.js?ver=5e580eb46a90c2b997e6" id="wp-i18n-js"></script> <script type="text/javascript" id="wp-i18n-js-after"> /* <![CDATA[ */ wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } ); /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/wp-content/mu-plugins/jetpack-14.4/jetpack_vendor/automattic/jetpack-assets/build/i18n-loader.js?minify=true&ver=becd7d9884bc1b331e45" id="wp-jp-i18n-loader-js"></script> <script type="text/javascript" id="wp-jp-i18n-loader-js-after"> /* <![CDATA[ */ wp.jpI18nLoader.state = {"baseUrl":"https://code.flickr.net/wp-content/languages/","locale":"en_US","domainMap":{"jetpack-admin-ui":"plugins/jetpack","jetpack-assets":"plugins/jetpack","jetpack-backup-pkg":"plugins/jetpack","jetpack-blaze":"plugins/jetpack","jetpack-boost-core":"plugins/jetpack","jetpack-boost-speed-score":"plugins/jetpack","jetpack-classic-theme-helper":"plugins/jetpack","jetpack-compat":"plugins/jetpack","jetpack-config":"plugins/jetpack","jetpack-connection":"plugins/jetpack","jetpack-explat":"plugins/jetpack","jetpack-external-media":"plugins/jetpack","jetpack-forms":"plugins/jetpack","jetpack-image-cdn":"plugins/jetpack","jetpack-import":"plugins/jetpack","jetpack-ip":"plugins/jetpack","jetpack-jitm":"plugins/jetpack","jetpack-licensing":"plugins/jetpack","jetpack-masterbar":"plugins/jetpack","jetpack-my-jetpack":"plugins/jetpack","jetpack-password-checker":"plugins/jetpack","jetpack-plugins-installer":"plugins/jetpack","jetpack-post-list":"plugins/jetpack","jetpack-protect-models":"plugins/jetpack","jetpack-protect-status":"plugins/jetpack","jetpack-publicize-pkg":"plugins/jetpack","jetpack-search-pkg":"plugins/jetpack","jetpack-stats":"plugins/jetpack","jetpack-stats-admin":"plugins/jetpack","jetpack-sync":"plugins/jetpack","jetpack-videopress-pkg":"plugins/jetpack","jetpack-waf":"plugins/jetpack","jetpack-wordads":"plugins/jetpack","woocommerce-analytics":"plugins/jetpack"},"domainPaths":{"jetpack-admin-ui":"jetpack_vendor/automattic/jetpack-admin-ui/","jetpack-assets":"jetpack_vendor/automattic/jetpack-assets/","jetpack-backup-pkg":"jetpack_vendor/automattic/jetpack-backup/","jetpack-blaze":"jetpack_vendor/automattic/jetpack-blaze/","jetpack-boost-core":"jetpack_vendor/automattic/jetpack-boost-core/","jetpack-boost-speed-score":"jetpack_vendor/automattic/jetpack-boost-speed-score/","jetpack-classic-theme-helper":"jetpack_vendor/automattic/jetpack-classic-theme-helper/","jetpack-compat":"jetpack_vendor/automattic/jetpack-compat/","jetpack-config":"jetpack_vendor/automattic/jetpack-config/","jetpack-connection":"jetpack_vendor/automattic/jetpack-connection/","jetpack-explat":"jetpack_vendor/automattic/jetpack-explat/","jetpack-external-media":"jetpack_vendor/automattic/jetpack-external-media/","jetpack-forms":"jetpack_vendor/automattic/jetpack-forms/","jetpack-image-cdn":"jetpack_vendor/automattic/jetpack-image-cdn/","jetpack-import":"jetpack_vendor/automattic/jetpack-import/","jetpack-ip":"jetpack_vendor/automattic/jetpack-ip/","jetpack-jitm":"jetpack_vendor/automattic/jetpack-jitm/","jetpack-licensing":"jetpack_vendor/automattic/jetpack-licensing/","jetpack-masterbar":"jetpack_vendor/automattic/jetpack-masterbar/","jetpack-my-jetpack":"jetpack_vendor/automattic/jetpack-my-jetpack/","jetpack-password-checker":"jetpack_vendor/automattic/jetpack-password-checker/","jetpack-plugins-installer":"jetpack_vendor/automattic/jetpack-plugins-installer/","jetpack-post-list":"jetpack_vendor/automattic/jetpack-post-list/","jetpack-protect-models":"jetpack_vendor/automattic/jetpack-protect-models/","jetpack-protect-status":"jetpack_vendor/automattic/jetpack-protect-status/","jetpack-publicize-pkg":"jetpack_vendor/automattic/jetpack-publicize/","jetpack-search-pkg":"jetpack_vendor/automattic/jetpack-search/","jetpack-stats":"jetpack_vendor/automattic/jetpack-stats/","jetpack-stats-admin":"jetpack_vendor/automattic/jetpack-stats-admin/","jetpack-sync":"jetpack_vendor/automattic/jetpack-sync/","jetpack-videopress-pkg":"jetpack_vendor/automattic/jetpack-videopress/","jetpack-waf":"jetpack_vendor/automattic/jetpack-waf/","jetpack-wordads":"jetpack_vendor/automattic/jetpack-wordads/","woocommerce-analytics":"jetpack_vendor/automattic/woocommerce-analytics/"}}; /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/_static/??/wp-includes/js/dist/vendor/wp-polyfill.min.js,/wp-includes/js/dist/url.min.js?m=1742994401j" ></script><script type="text/javascript" id="jetpack-instant-search-js-before"> /* <![CDATA[ */ var JetpackInstantSearchOptions=JSON.parse(decodeURIComponent("%7B%22overlayOptions%22%3A%7B%22colorTheme%22%3A%22light%22%2C%22enableInfScroll%22%3Atrue%2C%22enableFilteringOpensOverlay%22%3Atrue%2C%22enablePostDate%22%3Atrue%2C%22enableSort%22%3Atrue%2C%22highlightColor%22%3A%22%23FFC%22%2C%22overlayTrigger%22%3A%22submit%22%2C%22resultFormat%22%3A%22expanded%22%2C%22showPoweredBy%22%3Atrue%2C%22defaultSort%22%3A%22relevance%22%2C%22excludedPostTypes%22%3A%5B%5D%7D%2C%22homeUrl%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%22%2C%22locale%22%3A%22en-US%22%2C%22postsPerPage%22%3A10%2C%22siteId%22%3A185426273%2C%22postTypes%22%3A%7B%22post%22%3A%7B%22singular_name%22%3A%22Post%22%2C%22name%22%3A%22Posts%22%7D%2C%22page%22%3A%7B%22singular_name%22%3A%22Page%22%2C%22name%22%3A%22Pages%22%7D%2C%22attachment%22%3A%7B%22singular_name%22%3A%22Media%22%2C%22name%22%3A%22Media%22%7D%7D%2C%22webpackPublicPath%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-content%5C%2Fmu-plugins%5C%2Fjetpack-14.4%5C%2Fjetpack_vendor%5C%2Fautomattic%5C%2Fjetpack-search%5C%2Fbuild%5C%2Finstant-search%5C%2F%22%2C%22isPhotonEnabled%22%3Afalse%2C%22isFreePlan%22%3Afalse%2C%22apiRoot%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-json%5C%2F%22%2C%22apiNonce%22%3A%226532796dd2%22%2C%22isPrivateSite%22%3Afalse%2C%22isWpcom%22%3Afalse%2C%22hasOverlayWidgets%22%3Atrue%2C%22widgets%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_0%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_1%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_2%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22date_histogram_3%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%7D%5D%2C%22widgetsOutsideOverlay%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_4%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_5%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_6%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22date_histogram_7%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%7D%5D%2C%22hasNonSearchWidgets%22%3Afalse%2C%22preventTrackingCookiesReset%22%3Afalse%7D")); /* ]]> */ </script> <script type="text/javascript" src="https://code.flickr.net/wp-content/mu-plugins/jetpack-14.4/jetpack_vendor/automattic/jetpack-search/build/instant-search/jp-search.js?minify=false&ver=749aa845a5ccb16bc8a6" id="jetpack-instant-search-js"></script> <script type="text/javascript" src="//stats.wp.com/w.js?ver=202514" id="jp-tracks-js"></script> <script type="text/javascript" id="jetpack-stats-js-before"> /* <![CDATA[ */ _stq = window._stq || []; _stq.push([ "view", JSON.parse("{\"v\":\"ext\",\"blog\":\"185426273\",\"post\":\"0\",\"tz\":\"-7\",\"srv\":\"code.flickr.net\",\"hp\":\"vip\",\"j\":\"1:14.4.1\"}") ]); _stq.push([ "clickTrackerInit", "185426273", "0" ]); /* ]]> */ </script> <script type="text/javascript" src="https://stats.wp.com/e-202514.js" id="jetpack-stats-js" defer="defer" data-wp-strategy="defer"></script> <script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script> </body> </html>