CINXE.COM

<!DOCTYPE html>     <html lang="en-US">  <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width" /> <title> code.flickr.com | Page 2 </title> <link rel="profile" href="https://gmpg.org/xfn/11" /> <link rel="stylesheet" type="text/css" media="all" href="https://code.flickr.net/wp-content/themes/flickr-code/style.css?ver=20190507" /> <link rel="pingback" href="https://code.flickr.net/xmlrpc.php">  <meta name='robots' content='max-image-preview:large' /> <link rel='dns-prefetch' href='//stats.wp.com' /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Feed" href="https://code.flickr.net/feed/" /> <link rel="alternate" type="application/rss+xml" title="code.flickr.com » Comments Feed" href="https://code.flickr.net/comments/feed/" /> <script type="text/javascript"> window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/code.flickr.net\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.3.5"}}; /*! This file is auto-generated */ !function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83e\udef1\ud83c\udffb\u200d\ud83e\udef2\ud83c\udfff","\ud83e\udef1\ud83c\udffb\u200b\ud83e\udef2\ud83c\udfff")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings); </script> <style type="text/css"> img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 0.07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } </style> <link rel='stylesheet' id='all-css-0' href='https://code.flickr.net/wp-includes/css/dist/block-library/style.min.css?m=1732205989g' type='text/css' media='all' /> <style id='wp-block-library-inline-css'> .has-text-align-justify{text-align:justify;} </style> <style id='wp-block-library-theme-inline-css'> .wp-block-audio figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-audio figcaption{color:hsla(0,0%,100%,.65)}.wp-block-audio{margin:0 0 1em}.wp-block-code{border:1px solid #ccc;border-radius:4px;font-family:Menlo,Consolas,monaco,monospace;padding:.8em 1em}.wp-block-embed figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-embed figcaption{color:hsla(0,0%,100%,.65)}.wp-block-embed{margin:0 0 1em}.blocks-gallery-caption{color:#555;font-size:13px;text-align:center}.is-dark-theme .blocks-gallery-caption{color:hsla(0,0%,100%,.65)}.wp-block-image figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-image figcaption{color:hsla(0,0%,100%,.65)}.wp-block-image{margin:0 0 1em}.wp-block-pullquote{border-bottom:4px solid;border-top:4px solid;color:currentColor;margin-bottom:1.75em}.wp-block-pullquote cite,.wp-block-pullquote footer,.wp-block-pullquote__citation{color:currentColor;font-size:.8125em;font-style:normal;text-transform:uppercase}.wp-block-quote{border-left:.25em solid;margin:0 0 1.75em;padding-left:1em}.wp-block-quote cite,.wp-block-quote footer{color:currentColor;font-size:.8125em;font-style:normal;position:relative}.wp-block-quote.has-text-align-right{border-left:none;border-right:.25em solid;padding-left:0;padding-right:1em}.wp-block-quote.has-text-align-center{border:none;padding-left:0}.wp-block-quote.is-large,.wp-block-quote.is-style-large,.wp-block-quote.is-style-plain{border:none}.wp-block-search .wp-block-search__label{font-weight:700}.wp-block-search__button{border:1px solid #ccc;padding:.375em .625em}:where(.wp-block-group.has-background){padding:1.25em 2.375em}.wp-block-separator.has-css-opacity{opacity:.4}.wp-block-separator{border:none;border-bottom:2px solid;margin-left:auto;margin-right:auto}.wp-block-separator.has-alpha-channel-opacity{opacity:1}.wp-block-separator:not(.is-style-wide):not(.is-style-dots){width:100px}.wp-block-separator.has-background:not(.is-style-dots){border-bottom:none;height:1px}.wp-block-separator.has-background:not(.is-style-wide):not(.is-style-dots){height:2px}.wp-block-table{margin:0 0 1em}.wp-block-table td,.wp-block-table th{word-break:normal}.wp-block-table figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-table figcaption{color:hsla(0,0%,100%,.65)}.wp-block-video figcaption{color:#555;font-size:13px;text-align:center}.is-dark-theme .wp-block-video figcaption{color:hsla(0,0%,100%,.65)}.wp-block-video{margin:0 0 1em}.wp-block-template-part.has-background{margin-bottom:0;margin-top:0;padding:1.25em 2.375em} </style> <link rel='stylesheet' id='all-css-4' href='https://code.flickr.net/_static/??-eJzTLy/QzcxLzilNSS3WzyrWz01NyUxMzUnNTc0rQeEU5CRWphbp5qSmJyZX6uVm5uklFxfr6OPTDpRD5sM02efaGpobGxkZmFpaGgAARKUu4Q==' type='text/css' media='all' /> <style id='jetpack-sharing-buttons-style-inline-css'> .jetpack-sharing-buttons__services-list{display:flex;flex-direction:row;flex-wrap:wrap;gap:0;list-style-type:none;margin:5px;padding:0}.jetpack-sharing-buttons__services-list.has-small-icon-size{font-size:12px}.jetpack-sharing-buttons__services-list.has-normal-icon-size{font-size:16px}.jetpack-sharing-buttons__services-list.has-large-icon-size{font-size:24px}.jetpack-sharing-buttons__services-list.has-huge-icon-size{font-size:36px}@media print{.jetpack-sharing-buttons__services-list{display:none!important}}ul.jetpack-sharing-buttons__services-list.has-background{padding:1.25em 2.375em} </style> <style id='classic-theme-styles-inline-css'> /*! This file is auto-generated */ .wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none} </style> <style id='global-styles-inline-css'> body{--wp--preset--color--black: #000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--blue: #1982d1;--wp--preset--color--dark-gray: #373737;--wp--preset--color--medium-gray: #666;--wp--preset--color--light-gray: #e2e2e2;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flow > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-flow > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-flow > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-constrained > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-constrained > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > :where(:not(.alignleft):not(.alignright):not(.alignfull)){max-width: var(--wp--style--global--content-size);margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignwide{max-width: var(--wp--style--global--wide-size);}body .is-layout-flex{display: flex;}body .is-layout-flex{flex-wrap: wrap;align-items: center;}body .is-layout-flex > *{margin: 0;}body .is-layout-grid{display: grid;}body .is-layout-grid > *{margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;} .wp-block-navigation a:where(:not(.wp-element-button)){color: inherit;} :where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;} :where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;} .wp-block-pullquote{font-size: 1.5em;line-height: 1.6;} </style> <link rel='stylesheet' id='all-css-10' href='https://code.flickr.net/_static/??-eJzTLy/QTc7PK0nNK9EvyUjNTS3WLykHcipTc1LLUvP0i0sqc1L1kouLdfQxVablZCZnFwFFU1LxK0QxMiknPzm7GKTUPtfW0NzI0NDQxMDUEABaNTMI' type='text/css' media='all' /> <script type="text/javascript" src="https://code.flickr.net/_static/??-eJzTLy/QzcxLzilNSS3WzyrWT8ksLtEvS81LyS/SB0oV5OdUpmXm5ADVpBaV6OVm5ullFevo49FUlJqeClSbWJJfpFtUmleSmZtKjDYku/Aqz8jPzy6GqrDPtTU0NzYyMjC1tDTIAgApC0op" ></script><script src='https://code.flickr.net/wp-includes/js/dist/i18n.min.js?ver=7701b0c3857f914212ef' id='wp-i18n-js'></script> <script id="wp-i18n-js-after" type="text/javascript"> wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } ); </script> <script src='https://code.flickr.net/wp-content/mu-plugins/jetpack-13.1/jetpack_vendor/automattic/jetpack-assets/build/i18n-loader.js?minify=true&ver=ee939953aa2115e2ca59' id='wp-jp-i18n-loader-js'></script> <script id="wp-jp-i18n-loader-js-after" type="text/javascript"> wp.jpI18nLoader.state = {"baseUrl":"https://code.flickr.net/wp-content/languages/","locale":"en_US","domainMap":{"jetpack-admin-ui":"plugins/jetpack","jetpack-assets":"plugins/jetpack","jetpack-backup-pkg":"plugins/jetpack","jetpack-blaze":"plugins/jetpack","jetpack-boost-core":"plugins/jetpack","jetpack-boost-speed-score":"plugins/jetpack","jetpack-compat":"plugins/jetpack","jetpack-config":"plugins/jetpack","jetpack-connection":"plugins/jetpack","jetpack-forms":"plugins/jetpack","jetpack-google-fonts-provider":"plugins/jetpack","jetpack-idc":"plugins/jetpack","jetpack-image-cdn":"plugins/jetpack","jetpack-import":"plugins/jetpack","jetpack-ip":"plugins/jetpack","jetpack-jitm":"plugins/jetpack","jetpack-licensing":"plugins/jetpack","jetpack-my-jetpack":"plugins/jetpack","jetpack-password-checker":"plugins/jetpack","jetpack-plugins-installer":"plugins/jetpack","jetpack-post-list":"plugins/jetpack","jetpack-publicize-pkg":"plugins/jetpack","jetpack-search-pkg":"plugins/jetpack","jetpack-stats":"plugins/jetpack","jetpack-stats-admin":"plugins/jetpack","jetpack-sync":"plugins/jetpack","jetpack-videopress-pkg":"plugins/jetpack","jetpack-waf":"plugins/jetpack","jetpack-wordads":"plugins/jetpack"},"domainPaths":{"jetpack-admin-ui":"jetpack_vendor/automattic/jetpack-admin-ui/","jetpack-assets":"jetpack_vendor/automattic/jetpack-assets/","jetpack-backup-pkg":"jetpack_vendor/automattic/jetpack-backup/","jetpack-blaze":"jetpack_vendor/automattic/jetpack-blaze/","jetpack-boost-core":"jetpack_vendor/automattic/jetpack-boost-core/","jetpack-boost-speed-score":"jetpack_vendor/automattic/jetpack-boost-speed-score/","jetpack-compat":"jetpack_vendor/automattic/jetpack-compat/","jetpack-config":"jetpack_vendor/automattic/jetpack-config/","jetpack-connection":"jetpack_vendor/automattic/jetpack-connection/","jetpack-forms":"jetpack_vendor/automattic/jetpack-forms/","jetpack-google-fonts-provider":"jetpack_vendor/automattic/jetpack-google-fonts-provider/","jetpack-idc":"jetpack_vendor/automattic/jetpack-identity-crisis/","jetpack-image-cdn":"jetpack_vendor/automattic/jetpack-image-cdn/","jetpack-import":"jetpack_vendor/automattic/jetpack-import/","jetpack-ip":"jetpack_vendor/automattic/jetpack-ip/","jetpack-jitm":"jetpack_vendor/automattic/jetpack-jitm/","jetpack-licensing":"jetpack_vendor/automattic/jetpack-licensing/","jetpack-my-jetpack":"jetpack_vendor/automattic/jetpack-my-jetpack/","jetpack-password-checker":"jetpack_vendor/automattic/jetpack-password-checker/","jetpack-plugins-installer":"jetpack_vendor/automattic/jetpack-plugins-installer/","jetpack-post-list":"jetpack_vendor/automattic/jetpack-post-list/","jetpack-publicize-pkg":"jetpack_vendor/automattic/jetpack-publicize/","jetpack-search-pkg":"jetpack_vendor/automattic/jetpack-search/","jetpack-stats":"jetpack_vendor/automattic/jetpack-stats/","jetpack-stats-admin":"jetpack_vendor/automattic/jetpack-stats-admin/","jetpack-sync":"jetpack_vendor/automattic/jetpack-sync/","jetpack-videopress-pkg":"jetpack_vendor/automattic/jetpack-videopress/","jetpack-waf":"jetpack_vendor/automattic/jetpack-waf/","jetpack-wordads":"jetpack_vendor/automattic/jetpack-wordads/"}}; </script> <link rel="https://api.w.org/" href="https://code.flickr.net/wp-json/" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://code.flickr.net/xmlrpc.php?rsd" /> <meta name="generator" content="WordPress 6.3.5" /> <style>img#wpstats{display:none}</style> <style type="text/css" id="twentyeleven-header-css"> #site-title, #site-description { position: absolute; clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ clip: rect(1px, 1px, 1px, 1px); } </style> <link rel="stylesheet" type="text/css" id="wp-custom-css" href="https://code.flickr.net/?custom-css=e0fbe57d10" /></head> <body class="home blog paged custom-background wp-embed-responsive paged-2 jps-theme-flickr-code two-column right-sidebar"> <div class="skip-link"><a class="assistive-text" href="#content">Skip to primary content</a></div><div class="skip-link"><a class="assistive-text" href="#secondary">Skip to secondary content</a></div><div id="page" class="hfeed"> <header id="branding"> <hgroup> <h1 id="site-title"><span><a href="https://code.flickr.net/" rel="home">code.flickr.com</a></span></h1> <h2 id="site-description"></h2> </hgroup> <a href="https://code.flickr.net/"> <img src="https://wp.flickr.net/wp-content/uploads/sites/3/2012/09/code-flickr-com-drawn-header-grey-large.png" width="1000" height="157" alt="code.flickr.com" /> </a> <div class="only-search with-image"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> </form> </div> <nav id="access"> <h3 class="assistive-text">Main menu</h3> <div class="menu-menu-container"><ul id="menu-menu" class="menu"><li id="menu-item-2084" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2084"><a href="http://www.flickr.com/">Flickr</a></li> <li id="menu-item-2085" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2085"><a href="http://blog.flickr.net/">Flickr Blog</a></li> <li id="menu-item-2250" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2250"><a href="http://twitter.com/flickr">@flickr</a></li> <li id="menu-item-2086" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2086"><a href="http://twitter.com/flickrapi">@flickrapi</a></li> <li id="menu-item-2087" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2087"><a href="https://www.flickr.com/services/developer/">Developer Guidelines</a></li> <li id="menu-item-2088" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2088"><a href="http://www.flickr.com/services/api/">API</a></li> <li id="menu-item-2089" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2089"><a href="http://www.flickr.com/jobs/">Jobs</a></li> </ul></div> </nav> </header> <div id="main"> <div id="primary"> <div id="content" role="main"> <nav id="nav-above"> <h3 class="assistive-text">Post navigation</h3> <div class="nav-previous"><a href="https://code.flickr.net/page/3/" ><span class="meta-nav">←</span> Older posts</a></div> <div class="nav-next"><a href="https://code.flickr.net/" >Newer posts <span class="meta-nav">→</span></a></div> </nav> <article id="post-3319" class="post-3319 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2016/03/24/configuration-management-for-distributed-systems-using-github-and-cfg4j/" rel="bookmark">Configuration management for distributed systems (using GitHub and cfg4j)</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2016/03/24/configuration-management-for-distributed-systems-using-github-and-cfg4j/" title="4:39 am" rel="bookmark"><time class="entry-date" datetime="2016-03-24T04:39:31-07:00">March 24, 2016</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/norbertpotocki/" title="View all posts by norbertpotocki" rel="author">norbertpotocki</a></span></span> </div> </header> <div class="entry-content"> <p><i>Norbert Potocki, Software Engineer @ Yahoo Inc.</i></p> <h2><b>Warm up: <i>Why configuration management?</i></b></h2> <p>When working with large-scale software systems, configuration management becomes crucial; supporting non-uniform environments gets greatly simplified if you decouple code from configuration. While building complex software/products such as <a href="https://www.flickr.com/">Flickr</a>, we had to come up with a simple, yet powerful, way to manage configuration. Popular approaches to solving this problem include using configuration files or having a dedicated configuration service. Our new solution combines the extremely popular <a href="http://github.com/">GitHub</a> and <a href="http://www.cfg4j.org/">cfg4j</a> library, giving you a very flexible approach that will work with applications of any size.</p> <h4><b>Why should I decouple configuration from the code?</b></h4> <ul> <li>Faster configuration changes (e.g. flipping feature toggles): Configuration can simply be injected without requiring parts of your code to be reloaded and re-executed. Config-only updates tend to be faster than code deployment.</li> <li>Different configuration for different environments: Running your app on a laptop or in a test environment requires a different set of settings than production instance.</li> <li>Keeping credentials private: If you don’t have a dedicated credential store, it may be convenient to keep credentials as part of configuration. They usually aren’t supposed to be “public,” but the code still may be. Be a good sport and don’t keep credentials in a public GitHub repo. :)</li> </ul> <h2><b>Meet the Gang: <i>Overview of configuration management players</i></b></h2> <p>Let’s see what configuration-specific components we’ll be working with today:</p> <figure class="tmblr-full"><img decoding="async" class=" aligncenter" src="https://56.media.tumblr.com/33746f4780ff5e56d99c88e77e44e163/tumblr_inline_o47joy3QLn1t3d6pa_540.png" alt="image" /><figcaption><i> <small>Figure 1 – Overview of configuration management components</small><br /> </i></figcaption></figure> <p><i>Configuration repository and editor</i>: Where your configuration lives. We’re using <a href="https://git-scm.com/">Git</a> for storing configuration files and <a href="http://github.com/">GitHub</a> as an ad hoc editor.</p> <p><i>Push cache </i>: Intermediary store that we use to improve fetch speed and to ease load on GitHub servers.</p> <p><i>CD pipeline</i>: Continuous deployment pipeline pushing changes from repository to push cache, and validating config correctness.</p> <p><i>Configuration library</i>: Fetches configs from push cache and exposing them to your business logic.</p> <p><i>Bootstrap configuration </i>: Initial configuration specifying where your push cache is (so that library knows where to get configuration from).</p> <p>All these players work as a team to provide an end-to-end configuration management solution.</p> <h4><b>The Coach: <i>Configuration repository and editor</i></b></h4> <p>The first thing you might expect from the configuration repository and editor is ease of use. Let’s enumerate what that means:</p> <ul> <li>Configuration should be easy to read and write.</li> <li>It should be straightforward to add a new configuration set.</li> <li>You most certainly want to be able to review changes if your team is bigger than one person.</li> <li>It’s nice to see a history of changes, especially when you’re trying to fix a bug in the middle of the night.</li> <li>Support from popular IDEs – freedom of choice is priceless.</li> <li>Multi-tenancy support (optional) is often pragmatic.</li> </ul> <p>So what options are out there that may satisfy those requirements? The three very popular formats for storing configuration are <a href="http://yaml.org/">YAML</a>, Java Property files, and XML files. We use YAML – it is widely supported by multiple programming languages and IDEs, and it’s very readable and easy to understand, even by a non-engineer.</p> <p>We could use a dedicated configuration store; however, the great thing about files is that they can be easily versioned by version control tools like Git, which we decided to use as it’s widely known and proven.</p> <p>Git provides us with a history of changes and an easy way to branch off configuration. It also has great support in the form of GitHub which we use both as an editor (built-in support for YAML files) and collaboration tool (pull requests, forks, review tool). Both are nicely glued together by following the<a href="http://nvie.com/posts/a-successful-git-branching-model/"> Git flow branching model.</a> Here’s an example of a configuration file that we use:</p> <figure class="tmblr-full"><img decoding="async" class=" aligncenter" src="https://56.media.tumblr.com/23132e30bc9ea96dc7a6dd21d1125a7a/tumblr_inline_o47m8ne6iT1t3d6pa_540.png" alt="" /><figcaption><i> <small>Figure 2 – configuration file preview</small><br /> </i></figcaption></figure> <p>One of the goals was to make managing multiple configuration sets (execution environments) a breeze. We need the ability to add and remove environments quickly. If you look at the screenshot below, you’ll notice a “prod-us-east” directory in the path. For every environment, we store a separate directory with config files in Git. All of them have the exact same structure and only differ in YAML file contents.</p> <p>This solution makes working with environments simple and comes in very handy during local development or new production fleet rollout (see use cases at the end of this article). Here’s a sample config repo for a project that has only one “feature”:</p> <figure class="tmblr-full"><img decoding="async" class=" aligncenter" src="https://56.media.tumblr.com/06b1f785bb828923d6cd2b82f4b9f100/tumblr_inline_o47medL5NH1t3d6pa_540.png" alt="" /><figcaption><i><small>Figure 3 – support for multiple environments</small><br /> </i></figcaption></figure> <p>Some of the products that we work with at Yahoo have a very granular architecture with hundreds of micro-services working together. For scenarios like this, it’s convenient to store configurations for all services in a single repository. It greatly reduces the overhead of maintaining multiple repositories. We support this use case by having multiple top-level directories, each holding configurations for one service only.</p> <h4><b>The sprinter: <i>Push cache</i></b></h4> <p>The main role of push cache is to decrease the load put on the GitHub server and improve configuration fetch time. Since speed is the only concern here, we decided to keep the push cache simple: it’s just a key-value store. <a href="http://consul.io/">Consul</a> was our choice, in part because it’s fully distributed.</p> <p>You can install Consul clients on the edge nodes and they will keep being synchronized across the fleet. This greatly improves both the reliability and the performance of the system. If performance is not a concern, any key-value store will do. You can skip using push cache altogether and connect directly to Github, which comes in handy during development (see use cases to learn more about this).</p> <h4><b>The Manager: <i>CD Pipeline</i></b></h4> <p>When the configuration repository is updated, a CD pipeline kicks in. This fetches configuration, converts it into a more optimized format, and pushes it to cache. Additionally, the CD pipeline validates the configuration (once at pull-request stage and again after being merged to master) and controls multi-phase deployment by deploying config change to only 20% of production hosts at one time.</p> <h4><b>The Mascot: <i>Bootstrap configuration</i></b></h4> <p>Before we can connect to the push cache to fetch configuration, we need to know where it is. That’s where bootstrap configuration comes into play. It’s very simple. The config contains the hostname, port to connect to, and the name of the environment to use. You need to put this config with your code or as part of the CD pipeline. This simple yaml file binding Spring profiles to different Consul hosts suffices for our needs:</p> <figure class="tmblr-full"><img decoding="async" class=" aligncenter" src="https://56.media.tumblr.com/738020b5614fc06a26b85d2dc13789ad/tumblr_inline_o47k33QdkU1t3d6pa_540.png" alt="image" /><figcaption><i><small>Figure 4 – bootstrap configuration<br /> </small><br /> </i></figcaption></figure> <h4><b>The Cool Guy: <i>Configuration library</i></b></h4> <figure><img decoding="async" class=" aligncenter" src="https://56.media.tumblr.com/1370245b95cb98f542762a8040c6bf50/tumblr_inline_o47k4bgxl51t3d6pa_540.png" alt="image" /></figure> <p>The configuration library takes care of fetching the configuration from push cache and exposing it to your business logic. We use the library called <a href="http://www.cfg4j.org/">cfg4j</a> (“configuration for java”). This library re-loads configurations from the push cache every few seconds and injects them into configuration objects that our code uses. It also takes care of local caching, merging properties from different repositories, and falling back to user-provided defaults when necessary (read more at <a href="http://www.cfg4j.org/">http://www.cfg4j.org/</a>).</p> <p>Briefly summarizing how we use cfg4j’s features:</p> <ul> <li>Configuration auto-reloading: Each service reloads configuration every ~30 seconds and auto re-configures itself.</li> <li>Multi-environment support: for our multiple environments (beta, performance, canary, production-us-west, production-us-east, etc.).</li> <li>Local caching: Remedies service interruption when the push cache or configuration repository is down and also improves the performance for obtaining configs.</li> <li>Fallback and merge strategies: Simplifies local development and provides support for multiple configuration repositories.</li> <li>Integration with Dependency Injection containers – because we love DI!</li> </ul> <p>If you want to play with this library yourself, there’s plenty of examples both in<a href="http://www.cfg4j.org/"> its documentation</a> and<a href="https://github.com/cfg4j/cfg4j-sample-apps"> cfg4j-sample-apps Github repository</a>.</p> <h4><b>The Heavy Lifter: <i>Configurable code</i></b></h4> <p>The most important piece is business logic. To best make use of a configuration service, the business logic has to be able to re-configure itself in runtime. Here are a few rules of thumb and code samples:<b><br /> </b></p> <ul> <li>Use dependency injection for injecting configuration. This is how we do it using Spring Framework (see the bootstrap configuration above for host/port values):</li> </ul> <p><a href="https://gist.github.com/norbertpotocki/e91aa64b524592432630" rel="nofollow">https://gist.github.com/norbertpotocki/e91aa64b524592432630</a></p> <ul> <li>Use configuration objects to inject configuration instead of providing configuration directly – here’s where the difference is:</li> </ul> <p>Direct configuration injection (won’t reload as config changes)<br /> <a href="https://gist.github.com/norbertpotocki/eac0a927ca2df45c2a0b" rel="nofollow">https://gist.github.com/norbertpotocki/eac0a927ca2df45c2a0b</a></p> <p>Configuration injection via “interface binding” (will reload as config changes):<br /> <a href="https://gist.github.com/norbertpotocki/0c0b5b9aa9d11c06c937" rel="nofollow">https://gist.github.com/norbertpotocki/0c0b5b9aa9d11c06c937</a></p> <h2><b>The exercise: <i>Common use-cases (applying our simple solution)</i></b></h2> <h4><b>Configuration during development (local overrides)</b></h4> <p>When you develop a feature, a main concern is the ability to evolve your code quickly. A full configuration-management pipeline is not conducive to this. We use the following approaches when doing local development:<b><br /> </b></p> <ul> <li>Add a temporary configuration file to the project and use cfg4j’s <i>MergeConfigurationSource</i> for reading config both from the configuration store and your file. By making your local file a primary configuration source, you provide an override mechanism. If the property is found in your file, it will be used. If not, cfg4j will fall back to using values from configuration store. Here’s an example (reference examples above to get a complete code):</li> </ul> <p><a href="https://gist.github.com/norbertpotocki/289f3943249ea2813dcf" rel="nofollow">https://gist.github.com/norbertpotocki/289f3943249ea2813dcf</a></p> <ul> <li>Fork the configuration repository, make changes to the fork and use cfg4j’s GitConfigurationSource to access it directly (no push<br /> cache required):</li> </ul> <p><a href="https://gist.github.com/norbertpotocki/dacdcc6671a2158ded5e" rel="nofollow">https://gist.github.com/norbertpotocki/dacdcc6671a2158ded5e</a></p> <ul> <li>Set up your private push cache, point your service to the cache, and edit values in it directly.</li> </ul> <h4><b>Configuration defaults</b></h4> <p>When you work with multiple environments, some of them may share a configuration. That’s when using configuration defaults may be convenient. You can do this by creating a “default” environment and using cfg4j’s MergeConfigurationSource for reading config first from the original environment and then (as a fallback) from “default” environment.</p> <h4><b>Dealing with outages</b></h4> <p>Configuration repository, push cache, and configuration CD pipeline can experience outages. To minimize the impact of such events, it’s good practice to cache configuration locally (in-memory) after each fetch. cfg4j does that automatically.</p> <h4><b>Responding to incidents – ultra fast configuration updates (skipping configuration CD pipeline)</b></h4> <p>Tests can’t always detect all problems. Bugs leak to the production environment and at times it’s important to make a config change as fast as possible to stop the fire. If you’re using push cache, the fastest way to modify config values is to make changes directly within the cache. Consul offers a rich REST API and web ui for updating configuration in the key-value store.</p> <h4><b>Keeping code and configuration in sync</b></h4> <p>Verifying that code and configuration are kept in sync happens at the configuration CD pipeline level. One part of the continuous deployment process deploys the code into a temporary execution environment, and points it to the branch that contains the configuration changes. Once the service is up, we execute a batch of functional tests to verify configuration correctness.</p> <h2><b>The cool down: <i>Summary</i></b></h2> <p>The presented solution is the result of work that we put into building huge-scale photo-serving services. We needed a simple, yet flexible, configuration management system. Combining <a href="https://git-scm.com/">Git</a>, <a href="https://github.com/">Github</a>, <a href="http://consul.io/">Consul</a> and <a href="http://www.cfg4j.org/">cfg4j</a> provided a very satisfactory solution that we encourage you to try.</p> <p><i>I want to thank the following people for reviewing this article: <b>Bhautik Joshi, Elanna Belanger, Archie Russell</b>.</i></p> <p>PS. You can also follow me on <a href="https://twitter.com/norbert_potocki">Twitter</a>, <a href="https://github.com/norbertpotocki">GitHub</a>, <a href="https://www.linkedin.com/in/norbertpotocki">LinkedIn</a> or <a href="http://potocki.io/post/141230472743/configuration-management-for-distributed-systems">my private blog</a>.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> <article id="post-3281" class="post-3281 post type-post status-publish format-standard hentry category-labs category-metrics category-search tag-science"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/12/22/32-days-of-christmas/" rel="bookmark">The 32 Days Of Christmas!</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/12/22/32-days-of-christmas/" title="10:45 pm" rel="bookmark"><time class="entry-date" datetime="2015-12-22T22:45:03-08:00">December 22, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/aymanshamma/" title="View all posts by David A. Shamma" rel="author">David A. Shamma</a></span></span> </div> </header> <div class="entry-content"> <p><a data-flickr-embed="true" href="https://www.flickr.com/photos/kwl/6471898925/" title="LEGO City Advent Calendar - Day 7 by kennymatic, on Flickr"><img decoding="async" fetchpriority="high" src="https://farm8.staticflickr.com/7017/6471898925_0635fb3fff_z.jpg" width="640" height="426" alt="LEGO City Advent Calendar - Day 7"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script></p> <p>When you have thousands of photos, it can be hard to find the photo you’re looking for. Want to search for that Christmas cat you saw at last year’s party? And what if that party wasn’t on Christmas day, but sometime the week before? To help improve the search ranking and relevance of national, personal, and religious holiday photos, we first have to see when the photos were taken; when, for example, is the <em>Christmas season</em>?</p> <p>Understanding what people are looking for when they search for their own photos is an important part of improving Flickr. Earlier this year, we began a study (which will be published at <a href="http://chi2016.acm.org/wp/">CHI 2016</a> under the same name as this post) by trying to understand how people searched for their personal photos. We showed a group of 74 participants roughly 20 of their own photos on Flickr, and asked them what they’d put into the Flickr search box to find those photos. We did this a total of 1492 times.</p> <p>It turns out 12% of the time people used a <i>temporal</i> term in searches for their own photos, meaning a word connected to time in some way. These might include a year (2015), a month (January), a season (winter), or a holiday or special event (Thanksgiving, Eid al-Fitr, Easter, Passover, Burning Man). Often, however, the date and time on the photograph didn’t match the search term: the year would be wrong, or people would search for a photograph of snow the weekend after Thanksgiving with the word “winter,” despite the fact that winter doesn’t officially begin until December 21st in the U.S. So we wanted to understand that situation: <b>how often does <i>fall</i> feel like <i>winter</i>?</b></p> <p>To answer this, we mapped 78.8 million Flickr photos tagged with a season name to the date the photo was actually taken.</p> <p><img decoding="async" class="alignleft size-medium wp-image-3285" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?w=800" alt="Seasons Tagged by Date" width="800" height="215" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png 1368w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?resize=150,40 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?resize=800,215 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?resize=768,207 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?resize=1024,275 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f1-seasons.png?resize=500,135 500w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p>As you’d expect<b>, most of the photographs tagged with a season are taken during that season: 66% of photos tagged “winter” were taken between December 22 and March 20.</b> About 9% of search words are off by two seasons: photos tagged “summer” that were taken between December 21st and March 20th, for example. We expect this may reflect antipodean seasons: while most Flickr users are in the Northern Hemisphere, it doesn’t seem unreasonable that 5% of “summer” photographs might have been taken in the Southern Hemisphere. More interesting, we think, are the off-by-one cases, like fall photographs labeled as “winter,” where we believe that the photo represents the experience of winter, regardless of the objective reality of the calendar. For example, if it snows the day after Thanksgiving, it definitely feels like winter.</p> <p>On the topic of Thanksgiving, let’s look at photographs tagged “thanksgiving.”</p> <p><img decoding="async" class="alignleft size-medium wp-image-3287" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?w=800" alt="Percentage of Photos Tagged "Thanksgiving"" width="800" height="595" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png 1168w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?resize=150,111 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?resize=800,595 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?resize=768,571 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?resize=1024,761 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f2-thanksgiving.png?resize=404,300 404w" sizes="(max-width: 800px) 100vw, 800px" /><br /> The six days between November 22nd and 27th—the darkest blue area—cover 65% of the photos. Expanding that range to November 15–30th covers 83%. Expanding to all of November covers 85%, and including October (and thus Canadian Thanksgiving, in gray in early October) brings the total to 90%. But that means that 10% of all photos tagged “thanksgiving” are outside of this range. Every date in that image represents a total of a minimum of 40 photographs taken on that day between 2003 and 2014 inclusive, uploaded to Flickr and tagged “thanksgiving” with the only white spaces being days that don’t exist, like February 30th or April 31st. Manual verification of some of the public photos tagged “thanksgiving” on arbitrarily chosen dates finds these photographs tagged “thanksgiving” included pumpkins or turkeys, autumnal leaves or cornucopias—all images culturally associated with the holiday.</p> <p><b>Not all temporal search terms are quite so complicated;</b> some holidays are celebrated and photographed on a single day each year, like <a href="https://en.wikipedia.org/wiki/Canada_Day">Canada Day</a> (July 1st) or <a href="https://en.wikipedia.org/wiki/Boxing_Day">Boxing Day</a> (December 26th). While these holidays can be easily translated to date queries, other holidays have more complicated temporal patterns. Have a look at these lunar holidays.</p> <p><img decoding="async" loading="lazy" class="alignleft size-medium wp-image-3289" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?w=800" alt="Lunar Holidays Tagged by Date" width="800" height="683" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png 1368w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?resize=150,128 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?resize=800,683 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?resize=768,656 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?resize=1024,874 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f3-lunar.png?resize=351,300 351w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p>There are some events that occur on a lunar calendar like Chinese New Year, Easter, Eid (both al-Fitr and al-Adha), and Hanukkah. These events move around in a regular, algorithmically determinable, but sometimes complicated, way. Most of these holidays tend to oscillate as a leap calculation is added periodically to synchronize the lunar timing to the solar calendar. However Eids, on the <a href="https://en.wikipedia.org/wiki/Islamic_calendar">Hijri calendar</a>, have no such leap correction, and we see photos tagged “Eid” edge forward year after year.</p> <p>Some holidays and events, like birthdays, happen on every day of the week. But they’re often <i>celebrated</i>, and thus photographed, on Friday, Saturday, and Sunday:</p> <p><img decoding="async" loading="lazy" class="alignleft size-medium wp-image-3290" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png?w=800" alt="Day of the week tagged Birthday" width="800" height="155" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png 868w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png?resize=150,29 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png?resize=800,155 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png?resize=768,149 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f4-birthday.png?resize=500,97 500w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p>So to get back to our original question: when are photos tagged “Christmas” actually taken?</p> <p><img decoding="async" loading="lazy" class="alignleft size-medium wp-image-3291" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?w=800" alt="Days tagged with Christmas" width="800" height="595" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png 1168w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?resize=150,111 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?resize=800,595 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?resize=768,571 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?resize=1024,761 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/12/f5-christmas.png?resize=404,300 404w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p>As you can see, more photos tagged “Christmas” are taken on December 25th than on any other day (19%). Christmas Eve is a close second, at 12%. If you look at other languages, this difference practically goes away: 9.2% of photos tagged “Noel” are taken on Christmas Eve, and 9.6% are taken on Christmas; “navidad” photos are 11.3% on Christmas Eve and 12.0% on Christmas. But Christmas photos are taken throughout December. We can now set a threshold for a definition of Christmas: say if at least 1% of the photos tagged “Christmas” were taken on that day, we’d rank it more relevant. That means that every day from December 1st to January 1st hits that definition, with December 2nd barely scraping in. That makes…32 days of Christmas!</p> <p><em><strong>Merry Christmas and Happy Holidays</strong></em>—for all the holidays you celebrate and photograph.</p> <p>PS: <a href="https://www.flickr.com/jobs">Flickr is hiring</a>! <a href="https://labs.yahoo.com/careers">Labs is hiring</a>! Come join us!</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/labs/" rel="category tag">labs</a>, <a href="https://code.flickr.net/category/metrics/" rel="category tag">metrics</a>, <a href="https://code.flickr.net/category/search/" rel="category tag">search</a> </span> <span class="sep"> | </span> <span class="tag-links"> <span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="https://code.flickr.net/tag/science/" rel="tag">science</a> </span> </footer> </article> <article id="post-3238" class="post-3238 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/11/18/flickrs-experience-with-ios-9/" rel="bookmark">Flickr’s experience with iOS 9</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/11/18/flickrs-experience-with-ios-9/" title="6:00 am" rel="bookmark"><time class="entry-date" datetime="2015-11-18T06:00:47-08:00">November 18, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/rocirs/" title="View all posts by Rocir Santiago" rel="author">Rocir Santiago</a></span></span> </div> </header> <div class="entry-content"> <p>In the last couple of months, Apple has released new features as part of iOS 9 that allow a deeper integration between apps and the operating system. Among those features are Spotlight Search integration, Universal Links, and 3D Touch for iPhone 6S and iPhone 6S Plus.</p> <p>Here at Flickr, we have added support for these new features and we have learned a few lessons that we would love to share.</p> <h2><strong>Spotlight Search</strong></h2> <p>There are two different kinds of content that can be searched through Spotlight: the kind that you explicitly index, and the kind that gets indexed based on the state your app is in. To explicitly index content, you use Core Spotlight, which lets you index multiple items at once. To index content related to your app’s current state, you use NSUserActivity: when a piece of content becomes visible, you start an activity to make iOS aware of this fact. iOS can then determine which pieces of content are more frequently visited, and thus more relevant to the user. NSUserActivity also allows us to mark certain items as public, which means that they might be shown to other iOS users as well.</p> <p>For a better user experience, we index as much useful information as we can right off the bat. We prefetch all the user’s albums, groups, and people they follow, and add them to the search index using Core Spotlight. Indexing an item looks like this:</p> <pre class="brush: objc; title: ; notranslate" title=""> // Create the attribute set, which encapsulates the metadata of the item we're indexing CSSearchableItemAttributeSet *attributeSet = [[CSSearchableItemAttributeSet alloc] initWithItemContentType:(NSString *)kUTTypeImage]; attributeSet.title = photo.title; attributeSet.contentDescription = photo.searchableDescription; attributeSet.keywords = photo.keywords; attributeSet.thumbnailData = UIImageJPEGRepresentation(photo.thumbnail, 0.98); // Create the searchable item and index it. CSSearchableItem *searchableItem = [[CSSearchableItem alloc] initWithUniqueIdentifier:[NSString stringWithFormat:@&quot;%@/%@&quot;, photo.identifier, photo.searchContentType] domainIdentifier:@&quot;FLKCurrentUserSearchDomain&quot; attributeSet:attributeSet]; [[CSSearchableIndex defaultSearchableIndex] indexSearchableItems:@[ searchableItem ] completionHandler:^(NSError * _Nullable error) { if (error) { // Handle failures. } }]; </pre> <p>Since we have multiple kinds of data – photos, albums, and groups – we had to create an identifier that is a combination of its type and its actual model ID.</p> <p>Many users will have a large amount of data to be fetched, so it’s important that we take measures to make sure that the app still performs well. Since searching is unlikely to happen right after the user opens the app (that’s when we start prefetching this data, if needed), all this work is performed by a low-priority NSOperationQueue. If we ever need to fetch images to be used as thumbnails, we request it with low-priority <code>NSURLSessionDownloadTask</code>. These kinds of measures ensure that we don’t affect the performance of any operation or network request triggered by user actions, such as fetching new images and pages when scrolling through content.</p> <p>Flickr provides a huge amount of public content, including many amazing photos. If anybody searches for “Northern Lights” in Spotlight, shouldn’t we show them our best Aurora Borealis photos? For this public content – photos, public groups, tags and so on – we leverage NSUserActivity, with its new search APIs, to make it all searchable when viewed. Here’s an example:</p> <pre class="brush: objc; title: ; notranslate" title=""> CSSearchableItemAttributeSet *attributeSet = [[CSSearchableItemAttributeSet alloc] initWithItemContentType:(NSString *) kUTTypeImage]; // Setup attributeSet the same way we did before... // Set the related unique identifier, so it matches to any existing item indexed with Core Spotlight. attributeSet.relatedUniqueIdentifier = [NSString stringWithFormat:@&quot;%@/%@&quot;, photo.identifier, photo.searchContentType]; self.userActivity = [[NSUserActivity alloc] initWithActivityType:@&quot;FLKSearchableUserActivityType&quot;]; self.userActivity.title = photo.title; self.userActivity.keywords = [NSSet setWithArray:photo.keywords]; self.userActivity.webpageURL = photo.photoPageURL; self.userActivity.contentAttributeSet = attributeSet; self.userActivity.eligibleForSearch = YES; self.userActivity.eligibleForPublicIndexing = photo.isPublic; self.userActivity.requiredUserInfoKeys = [NSSet setWithArray:self.userActivity.userInfo.allKeys]; [self.userActivity becomeCurrent]; </pre> <p>Every time a user opens a photo, public group, location page, etc., we create a new NSUserActivity and make it current. The more often a specific activity is made current, the more relevant iOS considers it. In fact, the more often an activity is made current by any number of different users, the more relevant Apple considers it globally, and the more likely it will show up for other iOS users as well (provided it’s public).</p> <p>Until now we’ve only seen half the picture. We’ve seen how to index things for Spotlight search; when a user finally does search and taps on a result, how do we take them to the right place in our app? We’ll get to this a bit later, but for now suffice it to say that you’ll get a call to the method <code>application:continueUserActivity:restorationHandler:</code> to our application delegate.</p> <p>It’s important to note that if we wanted to make use of the <code>userInfo</code> in the <code>NSUserActivity</code>, iOS won’t give it back to you for free in this method. To get it, we have to make sure that we assigned an NSSet to the <code>requiredUserInfoKeys</code> property of our <code>NSUserActivity</code> when we created it. In their documentation, Apple also tells us that if you set the <code>webpageURL</code> property when <code>eligibleForSearch</code> is <code>YES</code>, you need to make sure that you’re pointing to the right web URL corresponding to your content, otherwise you might end up with duplicate results in Spotlight (Apple crawls your site for content to surface in Spotlight, and if it finds the same content at a different URL it’ll think it’s a different piece of content).</p> <h2><strong>Universal Links</strong></h2> <p>In order to support Universal Links, Apple requires that every domain supported by the app host an “apple-app-site-association” file at its root. This is a JSON file that describes which relative paths in your domains can be handled by the app. When a user taps a link from another app in iOS, if your app is able to handle that domain for a specific path, it will open your app and call <code>application:continueUserActivity:restorationHandler:</code>. Otherwise your application won’t be opened – Safari will handle the URL instead.</p> <pre class="brush: plain; title: ; notranslate" title=""> { &quot;applinks&quot;: { &quot;apps&quot;: [], &quot;details&quot;: { &quot;XXXXXXXXXX.com.some.flickr.domain&quot;: { &quot;paths&quot;: [ &quot;/&quot;, &quot;/photos/*&quot;, &quot;/people/*&quot;, &quot;/groups/*&quot; ] } } } } </pre> <p>This file has to be hosted on HTTPS with a valid certificate. Its MIME type needs to be “application/pkcs7-mime.” No redirects are allowed when requesting the file. If the only intent is to support Universal Links, no further steps are required. But if you’re also using this file to support Handoffs (introduced in iOS 8), then your file has to be CMS signed by a valid TLS certificate.</p> <p>In Flickr, we have a few different domains. That means that each one of flickr.com, <a href="http://www.flickr.com" rel="nofollow">http://www.flickr.com</a>, m.flickr.com and flic.kr must provide its own JSON association file, whether or not they differ. In our case, the flic.kr domain actually does support different paths, since it’s only used for short URLs; hence, its “apple-app-site-association” is different than the others.</p> <p>On the client side, only a few steps are required to support Universal Links. First, “Associated Domains” must be enabled under the Capabilities tab of the app’s target settings. For each supported domain, an entry “applinks:” entry must be added. Here is how it looks for Flickr:</p> <p><img decoding="async" loading="lazy" class="aligncenter size-full wp-image-3272" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/11/screen-shot-2015-10-28-at-2-00-59-pm.png" alt="Screen Shot 2015-10-28 at 2.00.59 PM" width="755" height="181" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/11/screen-shot-2015-10-28-at-2-00-59-pm.png 755w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/screen-shot-2015-10-28-at-2-00-59-pm.png?resize=150,36 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/screen-shot-2015-10-28-at-2-00-59-pm.png?resize=500,120 500w" sizes="(max-width: 755px) 100vw, 755px" /></p> <p>That is it. Now if someone receives a text message with a Flickr link, she will jump right to the Flickr app when she taps on it.</p> <h2><strong>Deep linking into the app</strong></h2> <p>Great! We have Flickr photos showing up as search results and Flickr URLs opening directly in our app. Now we just have to get the user to the proper place within the app. There are different entry points into our app, and we need to make the implementation consistent and avoid code duplication.</p> <p>iOS has been supporting deep linking for a while already and so has Flickr. To support deep linking, apps could register to handle custom URLs (meaning a custom scheme, such as myscheme://mydata/123). The website corresponding to the app could then publish links directly to the app. For every custom URL published on the Flickr website, our app translates it into a representation of the data to be shown. This representation looks like this:</p> <pre class="brush: objc; title: ; notranslate" title=""> @interface FLKRoute : NSObject @property (nonatomic) FLKRouteType type; @property (nonatomic, copy) NSString *identifier; @end </pre> <p>It describes the type of data to present, and a unique identifier for that type of data.</p> <pre class="brush: objc; title: ; notranslate" title=""> - (void)navigateToRoute:(FLKRoute *)route { switch (route.type) { case FLKRouteTypePhoto: // Navigate to photo screen break; case FLKRouteTypeAlbum: // Navigate to album screen break; case FLKRouteTypeGroup: // Navigate to group screen break; // ... default: break; } } </pre> <p>Now, all we have to do is to make sure we are able to translate both NSURLs and NSUserActivity objects into FLKRoute instances. For NSURLs, this translation is straightforward. Our custom URLs follow the same pattern as the corresponding website URLs; their paths correspond exactly. So translating both website URLs and custom URLs is a matter of using NSURLComponents to extract the necessary information to create the FLKRoute object.</p> <p>As for NSUserActivity objects passed into <code>application:continueUserActivity:restorationHandler:</code>, there are two cases. One arises when the NSUserActivity instance was used to index a public item in the app. Remember that when we created the NSUserActivity object we also assigned its <code>webpageURL</code>? This is really handy because it not only uniquely identifies the data we want to present, but also gives us a NSURL object, which we can handle the same way we handle deep links or Universal Links.</p> <p>The other case is when the NSUserActivity originated from a CSSearchableItem; we have some more work to do in this case. We need to parse the identifier we created for the item and translate it into a FLKRoute. Remember that our item’s identifier is a combination of its type and the model ID. We can decompose it and then create our route object. Its simplified implementation looks like this:</p> <pre class="brush: objc; title: ; notranslate" title=""> FLKRoute * FLKRouteFromSearchableItemIdentifier(NSString *searchableItemIdentifier) { NSArray *routeComponents = [searchableItemIdentifier componentsSeparatedByString:@&quot;/&quot;]; if ([routeComponents count] != 2) { // type + id return nil; } // Handle the route type NSString *searchableItemContentType = [routeComponents firstObject]; FLKRouteType type = FLKRouteTypeFromSearchableItemContentType(searchableItemContentType); // Get the item identifier NSString *itemIdentifier = [routeComponents lastObject]; // Build the route object FLKRoute *route = [FLKRoute new]; route.type = type; route.parameter = itemIdentifier; return route; } </pre> <p>Now we have all our bases covered and we’re sure that we’ll drop the user in the right place when she lands in our app. The final application delegate method looks like this:</p> <pre class="brush: objc; title: ; notranslate" title=""> - (BOOL)application:(nonnull UIApplication *)application continueUserActivity:(nonnull NSUserActivity *)userActivity restorationHandler:(nonnull void (^)(NSArray * __nullable))restorationHandler { FLKRoute *route; NSString *activityType = [userActivity activityType]; NSURL *url; if ([activityType isEqualToString:CSSearchableItemActionType]) { // Searchable item from Core Spotlight NSString *itemIdentifier = [userActivity.userInfo objectForKey:CSSearchableItemActivityIdentifier]; route = FLKRouteFromSearchableItemIdentifier(itemIdentifier); } else if ([activityType isEqualToString:@&quot;FLKSearchableUserActivityType&quot;] || [activityType isEqualToString:NSUserActivityTypeBrowsingWeb]) { // Searchable item from NSUserActivity or Universal Link url = userActivity.webpageURL; route = [url flk_route]; } if (route) { [self.router navigateToRoute:route]; return YES; } else if (url) { [[UIApplication sharedApplication] openURL:url]; // Fail gracefully return YES; } else { return NO; } } </pre> <h2><strong>3D Touch</strong></h2> <p>With the release of iPhone 6S and iPhone 6S Plus, Apple introduced a new gesture that can be used with your iOS app: 3D Touch. One of the coolest features it has brought is the ability to preview content before pushing it onto the navigation stack. This is also known as “peek and pop.”</p> <p>You can easily see how this feature is implemented in the native Mail app. But you won’t always have a simple UIView hierarchy like Mail’s UITableView, where a tap anywhere on a cell opens a UIViewController. Take Flickr’s notifications screen, for example:</p> <p><img decoding="async" loading="lazy" class="aligncenter size-medium wp-image-3273" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png?w=450" alt="4.0-04-core-five-notifications" width="450" height="800" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png 750w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png?resize=84,150 84w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png?resize=450,800 450w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png?resize=576,1024 576w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/4-0-04-core-five-notifications.png?resize=169,300 169w" sizes="(max-width: 450px) 100vw, 450px" /></p> <p>If the user taps on a photo in one of these cells, it will open the photo view. But if the user taps on another user’s name, it will open that user’s profile view. Previews of these UIViewControllers should be shown accordingly. But the “peek and pop” mechanism requires you to register a delegate on your UIViewController with <code>registerForPreviewingWithDelegate:sourceView:</code>, which means that you’re working in a much higher layer. Your UIViewController’s view might not even know about its subviews’ structures.</p> <p>To solve this problem, we used UIView’s method <code>hitTest:withEvent:</code>. As the documentation describes, it will give us the “farthest descendant of the receiver in the view hierarchy.” But not every hitTest will necessarily return the UIView that we want. So we defined a protocol, <code>FLKPeekAndPopTargetView</code>, that must be implemented by any UIView subclass that wants to support peeking and popping from it. That view is then responsible for returning the model used to populate the UIViewController that the user is trying to preview. If the view doesn’t implement this protocol, we query its superview. We keep checking for it until a UIView is found or there aren’t any more superviews available. This is how this logic looks:</p> <pre class="brush: objc; title: ; notranslate" title=""> + (id)modelAtLocation:(CGPoint)location inSourceView:(UIView*)sourceView // Walk up hit-test tree until we find a peek-pop target. UIView *testView = [sourceView hitTest:location withEvent:nil]; id model = nil; while(testView &amp;&amp; !model) { // Check if the current testView conforms to the protocol. if([testView conformsToProtocol:@protocol(FLKPeekAndPopTargetView)]) { // Translate location to view coordinates. CGPoint locationInView = [testView convertPoint:location fromView:sourceView]; // Get model from peek and pop target. model = [((id&lt;FLKPeekAndPopTargetView&gt;)testView) flk_peekAndPopModelAtLocation:locationInView]; } else { //Move up view tree to next view testView = testView.superview; } } return model; } </pre> <p>With this code in place, all we have to do is to implement <code>UIViewControllerPreviewingDelegate</code> methods in our delegate, perform the <code>hitTest</code> and take the model out of the <code>FLKPeekAndPopTargetView</code>‘s implementor. Here’s is the final implementation:</p> <pre class="brush: objc; title: ; notranslate" title=""> - (UIViewController *)previewingContext:(id&lt;UIViewControllerPreviewing&gt;)previewingContext viewControllerForLocation:(CGPoint)location { id model = [[self class] modelAtLocation:location inSourceView:previewingContext.sourceView]; UIViewController *viewController = nil; if ([model isKindOfClass:[FLKPhoto class]]) { viewController = // ... UIViewController that displays a photo. } else if ([model isKindOfClass:[FLKAlbum class]]) { viewController = // ... UIViewController that displays an album. } else if ([model isKindOfClass:[FLKGroup class]]) { viewController = // ... UIViewController that displays a group. } // ... return viewController; } - (void)previewingContext:(id&lt;UIViewControllerPreviewing&gt;)previewingContext commitViewController:(UIViewController *)viewControllerToCommit { [self.navigationController pushViewController:viewControllerToCommit animated:YES]; } </pre> <p>Last but not least, we added support for Quick Actions. Now the user has the ability to quickly jump into a specific section of the app just by pressing down on the app icon. Defining these Quick Actions statically in the Info.plist file is an easy way to implement this feature, but we decided to go one step further and define these options dynamically. One of the options we provide is “Upload Photo,” which takes the user to the asset picker screen. But if the user has Auto Uploadr turned on, this option isn’t that relevant, so instead we provide a different app icon menu option in its place.</p> <p>Here’s how you can create Quick Actions:</p> <pre class="brush: objc; title: ; notranslate" title=""> NSMutableArray&lt;UIApplicationShortcutItem *&gt; *items = [NSMutableArray array]; [items addObject:[[UIApplicationShortcutItem alloc] initWithType:@&quot;FLKShortcutItemFeed&quot; localizedTitle:NSLocalizedString(@&quot;Feed&quot;, nil)]]; [items addObject:[[UIApplicationShortcutItem alloc] initWithType:@&quot;FLKShortcutItemTakePhoto&quot; localizedTitle:NSLocalizedString(@&quot;Upload Photo&quot;, nil)] ]; [items addObject:[[UIApplicationShortcutItem alloc] initWithType:@&quot;FLKShortcutItemNotifications&quot; localizedTitle:NSLocalizedString(@&quot;Notifications&quot;, nil)]]; [items addObject:[[UIApplicationShortcutItem alloc] initWithType:@&quot;FLKShortcutItemSearch&quot; localizedTitle:NSLocalizedString(@&quot;Search&quot;, nil)]]; [[UIApplication sharedApplication] setShortcutItems:items]; </pre> <p>And this is how it looks like when the user presses down on the app icon:</p> <p><img decoding="async" loading="lazy" class="size-medium wp-image-3270 aligncenter" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png?w=450" alt="IMG_0344" width="450" height="800" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png 750w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png?resize=84,150 84w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png?resize=450,800 450w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png?resize=576,1024 576w, https://code.flickr.net/wp-content/uploads/sites/3/2015/11/img_0344.png?resize=169,300 169w" sizes="(max-width: 450px) 100vw, 450px" /></p> <p>Finally, we have to handle where to take the user after she selects one of these options. This is yet another place where we can make use of our <code>FLKRoute</code> object. To handle the app opening from a Quick Action, we need to implement <code>application:performActionForShortcutItem:completionHandler:</code> in the app delegate.</p> <pre class="brush: objc; title: ; notranslate" title=""> - (void)application:(UIApplication *)application performActionForShortcutItem:(UIApplicationShortcutItem *)shortcutItem completionHandler:(void (^)(BOOL))completionHandler { FLKRoute *route = [shortcutItem flk_route]; [self.router navigateToRoute:route]; completionHandler(YES); } </pre> <h2><strong>Conclusion</strong></h2> <p>There is a lot more to consider when shipping these features with an app. For example, with Flickr, there are various platforms the user could be using. It is important to make sure that the Spotlight index is up to date to reflect changes made anywhere. If the user has created a new album and/or left a group from his desktop browser, we need to make sure that those changes are reflected in the app, so the newly-created album can be found through Spotlight, but the newly-departed group cannot.</p> <p>All of this work should be totally opaque to the user, without hogging the device’s resources and deteriorating the user experience overall. That requires some considerations around threading and network priorities. Network requests for UI-relevant data should not be blocked because we have other network requests happening at the same time. With some careful prioritizing, using <code>NSOperationQueue</code> and <code>NSURLSession</code>, we managed to accomplish this with no major problems.</p> <p>Finally, we had to consider privacy, one of the pillars of Flickr. We had to be extremely careful not to violate any of the user’s settings. We’re careful to never publicly index private content, such as photos and albums. Also, photos marked “restricted” are not publicly indexed since they might expose content that some users might consider offensive.</p> <p>In this blog post we went into the basics of integrating iOS 9 Search, Universal Links, and 3D Touch in Flickr for iOS. In order to focus on those features, we simplified some of our examples to demonstrate how you could get started with them in your own app, and to show what challenges we faced.</p> <div class="hiring-banner"> <p class="group-photo"><a title="Flickr September 2014 by Bhautik Joshi, on Flickr" href="https://www.flickr.com/photos/captin_nod/14965686478/"><img decoding="async" loading="lazy" src="https://farm4.staticflickr.com/3893/14965686478_9278dbe39c_m.jpg" alt="Flickr September 2014" width="120" height="80" /></a></p> <p>Like this post? Have a love of online photography? Want to work with us? Flickr is hiring <strong>mobile, back-end and front-end engineers</strong>, in our San Francisco office. <strong>Find out more at <a href="https://www.flickr.com/jobs/">flickr.com/jobs</a></strong>.</p> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> <article id="post-3181" class="post-3181 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/25/perceptual-image-compression-at-flickr/" rel="bookmark">Perceptual Image Compression at Flickr</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/25/perceptual-image-compression-at-flickr/" title="5:35 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-25T17:35:47-07:00">September 25, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/archieflickr/" title="View all posts by Archie Russell" rel="author">Archie Russell</a></span></span> </div> </header> <div class="entry-content"> <p><i><span style="font-weight:400;">Archie Russell, Peter Norby, Saeideh Bakhshi</span></i></p> <p>At Flickr our users really care about image quality. They <i>also</i> care a lot about how responsive our apps are. Addressing both of these concerns simultaneously is challenging; higher quality images have larger file sizes and are slower to transfer. Slow transfers are especially noticeable on mobile devices. Flickr had historically aimed for high quality at the expense of larger files, but in late 2014 we implemented a method to <em>both</em> maintain image quality and decrease the byte-size of the images we serve to users. As image appearance is very important to our users, we performed an extensive user test before rolling this change out. Here’s how we did it.</p> <h2>Background: JPEG Quality Settings</h2> <div style="width: 355px" class="wp-caption alignright"><img decoding="async" loading="lazy" src="https://c1.staticflickr.com/1/755/21476411998_b47d8c2eb6_o.png" alt="" width="345" height="226" /><p class="wp-caption-text">Fig 1. JPEG settings vs file size for a test image.</p></div> <p>JPEG compression has several tuneable knobs. The <b><i>q-value</i></b> is the best known of these; it adjusts the level of spatial detail stored for fine details; a higher q-value typically keeps more detail. However, as q-value gets very close to 100, file size increases dramatically, usually without improving image appearance.</p> <p>If file size and app performance isn’t an issue, dialing up q-value is an easy way to get really nice-looking images; this is what Flickr has done in the past. And if appearance isn’t very important, dialing down q-value is a viable option. But if you want <i>both</i>, you’re kind of stuck. Additionally, q-value isn’t one-size-fits-all, some images look great at q-value 80 while others don’t.</p> <p>Another commonly adjusted setting is chroma-subsampling, which alters the amount of color information stored in a JPEG file. With a setting of 4:4:4, the two chroma (color) channels in a JPG have as much information as the luminance channel. In an image with a setting of 4:2:0, each chroma channel has only a quarter as much information as in an a 4:4:4 image.</p> <table> <tbody> <tr> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/573/21660491855_8874629a88_o.jpg" alt="" width="490" height="310" /> q=96, chroma=4:4:4 <em>(125KB)</em></td> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/752/21634483046_65f33afefc_o.jpg" alt="" width="490" height="310" />q=70, chroma=4:4:4 <em>(67KB)</em></td> </tr> <tr> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/781/21660537205_b4bc70dbd5_o.jpg" alt="" width="490" height="310" />q=96, chroma=4:2:0 <em>(62KB)</em></td> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/701/21472657030_c0f4906109_o.jpg" alt="" width="490" height="310" /> q=70, chroma=4:2:0 <em>(62KB)</em></td> </tr> </tbody> </table> <p>Table 1: JPEG stored at different quality and chroma levels. The upper left image is saved at high quality and chroma level; notice the color and detail in the folds of the red flag. The lower right image has the lowest quality; notice artifacts along the right edges of the red flag.</p> <h2>Perceptual JPEG Compression</h2> <p>Ideally we’d have an algorithm which automatically tuned all JPEG parameters to make a file smaller, but which would limit <i>perceptible</i> changes to the image. Technology exists that attempts to do this and can decrease image file size by 30-50%. This compression ratio is highly dependent on image content and dimensions.</p> <table> <tbody> <tr> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/714/21671860802_7c6a2e0c24_o.jpg" alt="" width="490" height="310" /></td> <td style="text-align:center;"><img decoding="async" loading="lazy" class="alignnone" src="https://c1.staticflickr.com/1/667/21495346570_61ce77defd_o.jpg" alt="" width="490" height="310" /></td> </tr> <tr> <td style="text-align:center;">compressed: 112KB</td> <td style="text-align:center;">non-compressed: 224KB</td> </tr> </tbody> </table> <p><span class="caption">Fig 2. <a href="https://c1.staticflickr.com/1/714/21671860802_7c6a2e0c24_o.jpg" target="_blank">Compressed</a> cropped JPEG is 50% smaller than <a href="https://c1.staticflickr.com/1/667/21495346570_61ce77defd_o.jpg" target="_blank">not-compressed</a> cropped JPEG, above, with no obvious defects. Compression ratio is similar for a <a href="https://c1.staticflickr.com/1/751/21642985766_73dc62fc53_o.jpg" target="_blank">compressed 2048-pixel </a>wide JPEG (475KB) of the entire scene and its <a href="https://c1.staticflickr.com/1/623/21481122700_01d36020e7_o.jpg" target="_blank">corresponding not-compressed</a> JPEG (897KB). </span></p> <p>We were pleased with perceptually compressed images in non-structured examinations. The compressed images were smaller and nearly indistinguishable from their sources. But we wanted to really quantify how well the technology worked before considering incorporating it into Flickr. The standard computational tools for evaluating compression, such as SSIM, are fairly simplistic and don’t do a great job at modeling how a user sees things. To really evaluate this technology had to use a better measure of perceptibility: human minds.</p> <p><img decoding="async" src="https://c1.staticflickr.com/1/752/21038236064_32950459f4_o.png" alt="The Gamified Taste Test" width="800" /></p> <p>To test whether our image compression would impact user perception of image quality, we put together a “taste test.” The taste test is constructed as a game with multiple rounds where users look at both compressed and uncompressed images. Users accumulate points the longer they play, and get more points for doing well at the game. We maintained a leaderboard to encourage participation and used only internal testers.The game’s test images came from a diverse collection of 250 images contributed by Flickr staff. The images came from a variety of cameras and included a number of subjects from photographers with varying skill levels.</p> <p><img decoding="async" loading="lazy" src="https://c1.staticflickr.com/1/751/21472835618_213bac1351_o.jpg" alt="sampling of images used in taste test" width="800" height="528" /><br /> <span class="caption">Fig 3. A sampling of images used in our taste test.</span></p> <p>In each round, our test code randomly select a test image, and present two variants of this image side by side. 50% of the time we present the user two identical images; the rest of the time we present one compressed image and one uncompressed image. We ask the tester if the two images look the same or different and we’d expect a user choosing randomly OR a user unable to distinguish the two cases would answer correctly about half the time. We randomly swap the location of the compressed images to compensate for user bias to the left or the right. If testers choose correctly, they are presented with a second question: “Which image did you prefer, and why?”</p> <p><img decoding="async" src="https://c2.staticflickr.com/6/5742/21038190014_37a5a38144_o.png" alt="two kittens in a video game" /><br /> <span class="caption">Fig 4. Screenshot of taste test.</span></p> <p>Our test displays images simultaneously to prevent testers noticing a longer load time for the larger, non-compressed image. The images are presented with either 320, 640, or 1600 pixels on their longest side. The 320 & 640px images are shown for 12 seconds before being dimmed out. The intent behind this detail is to represent how real users interact with our images. The 1600px images stay on screen for 20 seconds, as we expect larger images to be viewed for longer periods of time by real users. We award 100 points per round, regardless of whether a tester chose correctly and also award a bonus of 400 points when a tester correctly identifies whether images were identical or different. We update the tester’s score every five tests so that the user perceives an increasing score without being rewarded immediately for any particular behavior.</p> <h2>Taste Test Outcome and Deployment</h2> <p>We ran our taste test for two weeks and analyzed our results. Although we let users play as long as they liked, we skipped the first result per user as a “warm-up” and considered only the subsequent ten results, this limited the potential for users training themselves to spot compression artifacts. We disregarded users that had fewer than eleven results.</p> <table> <tbody> <tr> <td></td> <td></td> </tr> </tbody> <tbody> <tr> <td style="text-align:center;"><b>images</b></td> <td style="text-align:center;"><b>total results</b></td> <td style="text-align:center;"><b># labeled “identical” by tester</b></td> <td style="text-align:center;"><b>% labeled “identical” by tester</b></td> </tr> <tr> <td>two identical images</td> <td style="text-align:center;"><span style="font-weight:400;">368</span></td> <td style="text-align:center;"><span style="font-weight:400;">253</span></td> <td style="text-align:center;"><span style="font-weight:400;">68.8%</span></td> </tr> <tr> <td>one compressed, one non-compressed</td> <td style="text-align:center;"><span style="font-weight:400;">352</span></td> <td style="text-align:center;"><span style="font-weight:400;">238</span></td> <td style="text-align:center;"><span style="font-weight:400;">67.6%</span></td> </tr> </tbody> </table> <p><span class="caption">Table 2. Taste test results. Testers select “identical” at nearly the same rate, whether the input is identical or not.</span></p> <p>When our testers were presented with two <i>identical</i> images, they thought the images were identical only 68.8% of the time(!), and when presented with a compressed image next to a non-compressed image, our testers thought the images were identical slightly less often: 67.6% of the time. This difference was small enough for us, and our statisticians told us it was statistically insignificant. Our image pairs were so similar that multiple testers thought all images were identical and reported that the test system was buggy. We inspected the images most often labeled different, and found no significant artifacts in the compressed versions.</p> <p>So even in this side-by-side test, perceptual image compression is just barely noticeable when images are presented side-by-side. As the Flickr website wouldn’t ever show compressed and uncompressed images at the same time, and the use of compression had large benefits in storage footprint and site performance, we elected to go forward.</p> <p>At the beginning of 2014 we silently rolled out perceptual-based compression on our image thumbnails (we don’t alter the “original” images uploaded by our users). The slight changes to image appearance went unnoticed by users, but user interactions with Flickr became much faster, especially for users with slow connections, while our storage footprint became much smaller. This was a best-case scenario for us.</p> <p>Evaluating perceptual compression was a considerable task, but it gave the confidence we needed to apply this compression in production to our users. This marked the first time Flickr had adjusted image settings in years, and, it was fun.<br /> <img decoding="async" src="https://c1.staticflickr.com/1/633/21670107841_8b1a09c0a8_o.png" alt="High Score List" width="800" /><br /> <span class="caption">Fig 5. Taste test high score list</span></p> <h2>Epilogue</h2> <p>After eighteen months of perceptual compression at Flickr, we adjusted our settings slightly to shrink images an additional 15%. For our users on mobile devices, 15% fewer bytes per image makes for a much more responsive experience.We had run a taste test on this newer setting and users were were able to spot our compression slightly more often than with our original settings. When presented a pair of identical images, our testers declared these images identical 65.2% of the time, when presented with different images, of our testers declared the images identical 62% of the time. It wasn’t as imperceptible as our original approach, but, we decided it was close enough to roll out.</p> <p>Boy were we wrong! A few very vocal users spotted the compression and didn’t like it at all. The Flickr Help Forum had a very lively thread which <a href="http://petapixel.com/2015/06/08/flickr-decreased-quality-and-increased-compression-and-users-arent-happy/">Petapixel picked up</a>. We <span style="text-decoration:line-through;">beat our heads against the wall</span> considered our options and came up with a middle path between our initial and follow-on approaches, giving us smaller, faster-to-load files while still maintaining the appearance our users expect.</p> <p>Through our use of perceptual compression, combined with our use of <a href="http://code.flickr.net/2015/06/25/real-time-resizing-of-flickr-images-using-gpus/">on-the-fly resize</a> and <a href="http://yahooeng.tumblr.com/post/116391291701/yahoo-cloud-object-store-object-storage-at">COS</a>, we’ve been able to decrease our storage footprint dramatically, while simultaneously improving user experience. It’s a win all around but we’re not done yet — we still have a few tricks up our sleeves.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> <article id="post-3153" class="post-3153 post type-post status-publish format-standard hentry category-hadoop category-infrastructure category-kittens tag-hadoop tag-hbase tag-lambda-architecture tag-magic-view tag-pig-latin tag-storm"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/03/powering-flickrs-magic-view/" rel="bookmark">Powering Flickr’s Magic view by fusing bulk and real-time compute</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/03/powering-flickrs-magic-view/" title="5:30 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-03T17:30:21-07:00">September 3, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/bhautikj/" title="View all posts by Bhautik Joshi" rel="author">Bhautik Joshi</a></span></span> </div> </header> <div class="entry-content"> <blockquote> <h1>Try it for yourself!</h1> <p>You can try out Flickr’s Magic View on your own photos <a href="https://www.flickr.com/cameraroll">here</a>, and you can download a working code sample of the simplified lambda architecture here: <a href="https://github.com/yahoo/simplified-lambda">https://github.com/yahoo/simplified-lambda</a></p></blockquote> <h1>Introduction</h1> <p><span style="font-weight:400;">In this post we’re going to talk about how we came up with a novel revision of the Lambda Architecture for fusing large-scale bulk compute with streaming compute to power Flickr’s Magic View. We were able to create a responsive, real time database operating at a scale of tens of billions of records, with tens to hundreds of millions of records updated per day.</span> We turned to <a href="http://yahoohadoop.tumblr.com/">Yahoo’s Hadoop stack</a> to find a way to build this at the massive scale we needed.</p> <p><img decoding="async" loading="lazy" class="aligncenter size-medium wp-image-3156" style="text-align:center;" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?w=800" alt="Magic View" width="800" height="593" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg 1600w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=150,111 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=800,593 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=768,569 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=1024,758 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=1536,1138 1536w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/magic_view_overview.jpg?resize=405,300 405w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 1. Magic View in action</em></strong></p> <h2>Motivation: the Magic View</h2> <p><span style="font-weight:400;">Flickr’s Magic View takes the hassle out of organizing your photos by applying our computer-vision technology to automatically recognize objects or styles in your photos and present them to you in the Camera Roll’s scrolling view. This all happens in real time </span><span style="font-weight:400;">—</span><span style="font-weight:400;"> as soon as a photo is uploaded, it is categorized and placed into the Magic View.</span></p> <h2>Aggregating computer vision tags</h2> <p><span style="font-weight:400;">When a photo is uploaded, it is processed by a computer vision pipeline to generate a set of computer vision</span><i><span style="font-weight:400;"> tags</span></i><span style="font-weight:400;">, which are text labels of the contents of the image. We already had an existing architecture for stream computation of tags on upload, but to implement the Magic View, we needed to maintain per-user reverse indexes and some aggregations of the tags. And we needed to make sure all the data was consistent </span><span style="font-weight:400;">—</span><span style="font-weight:400;"> if a photo was added, removed or updated these indexes and aggregations would have to be updated to reflect this. Finally, we needed to initialize the system with tags for 12 billion photos and videos and run periodic backfills (every time we improved our computer vision algorithms and to cover cases where the stream compute missed images).</span></p> <h2>The Problem</h2> <p><span style="font-weight:400;">We initially computed a snapshot of the Magic View indexes and aggregations using map-reduce (via </span><a href="http://oozie.apache.org/"><span style="font-weight:400;">Apache Oozie</span></a><span style="font-weight:400;"> and </span><a href="http://pig.apache.org/"><span style="font-weight:400;">Apache Pig</span></a><span style="font-weight:400;">), and we were happy with the quick turnaround time (about 7 hours). We considered updating Magic View as a daily batch job, but soon realized this would not give our users the responsive, “live” experience we wanted. So, we built a streaming data layer using </span><a href="https://storm.apache.org/"><span style="font-weight:400;">Apache Storm</span></a><span style="font-weight:400;"> and were soon able to update the categories in Magic View in real-time.</span></p> <p><span style="font-weight:400;">The next time we needed to run a backfill, we explored using this streaming layer to load the data. Unfortunately, the overhead of the read-modify-write process was simply too much for a load of this size — after kicking off the process we estimated it would take <i><span style="font-weight:400;">28 days</span></i><span style="font-weight:400;"> this way — much longer than the seven hours we had achieved with a bulk load.</span></span></p> <p><span style="font-weight:400;">Twenty-eight days was a non-starter – we realized we needed a way to update our bulk aggregations independently of the real-time data streaming in. Solving this problem is how we arrived at our revision to Lambda Architecture. Before digging into the solution, let’s do a quick review of the Lambda Architecture. If you’re already familiar with it, you can skip this next section.</span></p> <h2>The Lambda Architecture</h2> <p>We’ll start with Nathan Marz’s book ‘<a href="https://www.manning.com/books/big-data">Big Data</a>’, which proposes the database concept of ‘Lambda Architecture.’ In his analysis, he states that a database query can be represented as a function – Query – which operates on all the data:</p> <pre style="text-align:center;">result = Query(all data)</pre> <p><span style="font-weight:400;">In the Lambda architecture, a traditional database is replaced with both a </span><i><span style="font-weight:400;">real time</span></i><span style="font-weight:400;"> and a </span><i><span style="font-weight:400;">bulk</span></i><span style="font-weight:400;"> database. Then query function becomes a “combiner” function of independent queries to each database:</span></p> <pre style="text-align:center;">result = Combiner(Query(real time data) + Query(bulk data))</pre> <p><span style="font-weight:400;">An example of a typical Lambda Architecture is shown in figure 2. It is powered by an append-only queue for its system of record, which is fed by a real time stream of events. Periodically, all the data in the queue is fed into a bulk computation which pre-processes the data to optimize it for queries, and stores these aggregations in a bulk compute database. The real time event stream drives a stream computer, which processes the incoming events into real time aggregations. A query then goes via a query combiner, which queries both the bulk and real time databases, computes the combination, and stores the result.</span></p> <p><img decoding="async" loading="lazy" class="aligncenter size-medium wp-image-3157" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?w=800" alt="Typical Lambda Architecture" width="800" height="339" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png 1039w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=150,64 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=800,339 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=768,325 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=1024,434 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/lambda_basic.png?resize=500,212 500w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 2. Typical Lambda Architecture</em></strong></p> <p><span style="font-weight:400;">While relatively new, Lambda Architecture has enjoyed popularity and a number of concrete implementations have been built. Some significant examples are the distributed analytics platform </span><a href="http://druid.io/"><span style="font-weight:400;">druid</span></a><span style="font-weight:400;">, </span><a href="https://github.com/twitter/summingbird"><span style="font-weight:400;">Twitter’s Summingbird</span></a><span style="font-weight:400;">, and </span><a href="https://github.com/velvia/FiloDB"><span style="font-weight:400;">FiloDB</span></a><span style="font-weight:400;">. These implementations conveniently abstract away the databases behind the query combiner. </span></p> <p><span style="font-weight:400;">A significant advantage with this style of architecture is robustness and fault-tolerance via eventual consistency. If a piece of data is skipped in the real time compute there is a guarantee that it will eventually appear in the bulk compute database.</span></p> <p><span style="font-weight:400;">Criticism of the Lambda Architecture has centred around the </span><a href="http://radar.oreilly.com/2014/07/questioning-the-Lambda-architecture.html"><span style="font-weight:400;">complicated nature of the combiner</span></a><span style="font-weight:400;">. The combiner incurs a developer and systems cost from the need to maintain two different databases. It can be challenging to make sure both systems give the same result. Merging the two queries can become complicated, and finally, more points of failure may be introduced.</span></p> <h2>The “Ah-ha” Moment</h2> <p><span style="font-weight:400;">Back to the problem. The data access layer we used for streaming compute uses the atomic read-modify-write pattern to ensure we write consistent data, one record-at-a-time to </span><a href="http://hbase.apache.org/"><span style="font-weight:400;">Apache HBase</span></a><span style="font-weight:400;"> (a BigTable-style, non-relational database). Again, since this pattern was so much slower in the backfill case we needed to figure out how to get both consistent updates for streaming </span><i><span style="font-weight:400;">and </span></i><span style="font-weight:400;"> fast loads of the full dataset. Since our bulk data was static, we realized that if we relaxed the consistency constraint we could just run a fast, streaming, write-only load of the bulk data, bringing the load time back down to hours instead of days.</span></p> <p><span style="font-weight:400;">But how could we get around the consistency requirements? We didn’t want a bulk load to clobber data being written from the real time compute process. The insight was that we could just write bulk and streaming data to different column families in the same HBase row. So we added the concept of </span><i><span style="font-weight:400;">real time columns</span></i><span style="font-weight:400;"> and </span><i><span style="font-weight:400;">bulk columns</span></i><span style="font-weight:400;"> in a single row. Basically, bulk loads write to one set of columns and real time writes go to a different set of columns. Since HBase columns are sparse and data is updated relatively slowly we don’t pay much in storage or IO.</span></p> <p><span style="font-weight:400;">We could now simplify the equation back to:</span></p> <pre style="text-align:center;">result = Combiner(Query(data))</pre> <p><span style="font-weight:400;">The two sets of columns are managed separately by the real time and bulk subsystems. At query time, we perform a single fetch using the HBase API to get </span><i><span style="font-weight:400;">both</span></i><span style="font-weight:400;"> the bulk and real time data. A separate combiner process assembles the final result. </span></p> <h1>Implementation</h1> <p><img decoding="async" loading="lazy" class="aligncenter size-medium wp-image-3159" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?w=800" alt=" Magic View backend system overview" width="800" height="347" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png 1133w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=150,65 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=800,347 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=768,333 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=1024,444 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/09/simplified_lambda.png?resize=500,217 500w" sizes="(max-width: 800px) 100vw, 800px" /></p> <p><strong><em>Figure 3. Magic View Architecture</em></strong></p> <p><span style="font-weight:400;">Figure 3 shows an overview of the system and our enhanced Lambda architecture. For the purposes of this discussion, a convenient abstraction is to consider that each row in the HBase table represents the current state of a given photo. The combiner stage is abstracted into a single Java process, which collects data from HBase and runs transformations on the data and sends it to a </span><a href="http://redis.io/"><span style="font-weight:400;">Redis</span></a><span style="font-weight:400;"> cache which is used by the serving layer for the site.</span></p> <h2>Consistency on read in HBase — the combiner</h2> <p><span style="font-weight:400;">We have two sets of columns to go with each row in HBase: bulk and real time. The combiner determines the final value for each attribute at read. In the case where data exists for real time but not for bulk (or vice versa) then there is only one value to choose. In the case where they both exist we always choose the real time value. This keeps the combiner very simple and fast. </span></p> <p><span style="font-weight:400;">There is a trick though – whenever we do a backfill, we may need to repair the row since the backfill data may be newer than any real time data that is already present. It turns out this slows down the backfill from seven hours to about 14 — still far faster than loading with read-modify-write.</span></p> <h2>Production throughput</h2> <p><span style="font-weight:400;">At scale, this architecture has been able to keep up very comfortably with production load. We can simultaneously run backfills to HBase and serve user information at the same time without impacting latency or the user experience.</span></p> <h2>User experience</h2> <p><span style="font-weight:400;">An important measure for how the system works is how the viewer perceives it. The slowest part of the system is paging data from HBase into the serving cache; median time for above-the-fold latency – i.e. enough data is available to render the page – is around 10ms. </span></p> <h2>Future directions</h2> <p><span style="font-weight:400;">Our experience has been very positive so far with Magic View and we’re looking at how we might enable users to browse their photos in other dimensions (location or color for example). Early tests have shown that building an OLAP or data cube in this architecture is certainly possible but it’s less clear that it will scale well.</span></p> <p><i><span style="font-weight:400;">Contributors: Peter Welch, Bhautik Joshi, Hugo Haas, Srinivasan Singanallur, Ayan Ray, Pierre Garrigues, Ben Firestone, Sai Madhavan, Tim Miller</span></i></p> <p><em>Thanks to Nathan Marz for reviewing this post.</em></p> <div class="hiring-banner"> <p class="group-photo"><a title="Flickr September 2014 by Bhautik Joshi, on Flickr" href="https://www.flickr.com/photos/captin_nod/14965686478/"><img decoding="async" loading="lazy" src="https://farm4.staticflickr.com/3893/14965686478_9278dbe39c_m.jpg" alt="Flickr September 2014" width="120" height="80" /></a></p> <p>Like this post? Have a love of online photography? Want to work with us? Flickr is hiring <strong>mobile, back-end and front-end engineers</strong>, in our San Francisco office. <strong>Find out more at <a href="https://www.flickr.com/jobs/">flickr.com/jobs</a></strong>.</p> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/hadoop/" rel="category tag">hadoop</a>, <a href="https://code.flickr.net/category/infrastructure/" rel="category tag">infrastructure</a>, <a href="https://code.flickr.net/category/kittens/" rel="category tag">kittens</a> </span> <span class="sep"> | </span> <span class="tag-links"> <span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="https://code.flickr.net/tag/hadoop/" rel="tag">hadoop</a>, <a href="https://code.flickr.net/tag/hbase/" rel="tag">hbase</a>, <a href="https://code.flickr.net/tag/lambda-architecture/" rel="tag">lambda architecture</a>, <a href="https://code.flickr.net/tag/magic-view/" rel="tag">magic view</a>, <a href="https://code.flickr.net/tag/pig-latin/" rel="tag">pig latin</a>, <a href="https://code.flickr.net/tag/storm/" rel="tag">storm</a> </span> </footer> </article> <article id="post-3114" class="post-3114 post type-post status-publish format-standard hentry category-uncategorized tag-api tag-caching tag-data-model tag-eviction tag-model"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/09/01/the-data-freshener/" rel="bookmark">The Data Freshener</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/09/01/the-data-freshener/" title="4:26 pm" rel="bookmark"><time class="entry-date" datetime="2015-09-01T16:26:49-07:00">September 1, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/ericsoco/" title="View all posts by ericsoco" rel="author">ericsoco</a></span></span> </div> </header> <div class="entry-content"> <div style="width:340px;"> <a href="https://www.flickr.com/photos/hellomokona/956099245/">https://www.flickr.com/photos/hellomokona/956099245/</a><br /> <em>So fresh</em> </div> <p> </p> <h2>Change</h2> <p>You may have noticed some changes in Flickr a couple months back. Like, half the site changed. 95% even, by some metrics. Some say CHANGE IT BACK! while others welcome change. Whatever your thoughts, the changes are here, and they mean things. For example, they mean new visual design and better usability. They mean a faster site. Unfortunately, up until recently, they also meant more stale data. Yuck.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/spcbrass/4388396268/" title="Change by spcbrass, on Flickr"><img decoding="async" loading="lazy" src="https://farm3.staticflickr.com/2759/4388396268_87224c6556_z.jpg" width="640" height="480" alt="Change"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Change</em><br />   </div> <p>Why? What? Well…here’s the deal. We have a new-ish frontend stack we’ve been using for the past couple years now. It’s an <a href="http://isomorphic.net/javascript">isomorphic</a> <a href="https://en.wikipedia.org/wiki/Single-page_application">single-page application</a>, runs on <a href="http://nodejs.org">node.js</a>, and is generally awesome. We call it Reboot.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/wafer/6897507098/" title="hi there / i am the computer by waferbaby, on Flickr"><img decoding="async" loading="lazy" src="https://farm6.staticflickr.com/5466/6897507098_78d4c5a056_z.jpg" width="640" height="425" alt="hi there / i am the computer"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Reboot</em><br />   </div> <p>In the World of Reboot, we treat data with kid gloves. We <3 data. We never want to give it up, never want to let it down. Once we pull data from our APIs, we store the fetched data in your browser so that we don’t have to fetch it again the next time it’s needed. This means faster page loads and faster navigation, and less API traffic (and thus a more stable and scalable API). The data cached in your browser exists as long as the current Reboot session — until you refresh or leave Reboot for a non-Rebooted page.</p> <p>However, this also meant that data could become stale. You change the date taken of your photo, someone else adds a comment, you navigate to a page with cached data…and you don’t see the changes. Wat? Yeah. So, this was not a huge problem until we moved lots of pages onto Reboot in the beginning of May. From that point forward, most Flickr user sessions have spent their entirety on Reboot, feeding off the same stale loaves of cached data.</p> <div style="width:480px;"> <a href="https://www.flickr.com/photos/recyclethis/157108084/">https://www.flickr.com/photos/recyclethis/157108084/</a><br /> <em>Staleness</em><br />   </div> <h2>The thinking (design / prototypes)</h2> <p>We considered a number of possibilities for freshening up data during a user session. A brief history of the strategies we sampled, and their results:</p> <h3>1. Refresh on update</h3> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/the_family_farm/2571858493/" title="Ice Tea by MzScarlett / A.K.A. Michelle, on Flickr"><img decoding="async" loading="lazy" src="https://farm4.staticflickr.com/3075/2571858493_34a1f75ac3_z.jpg" width="640" height="480" alt="Ice Tea"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>The first stab focused on updating data locally after it was changed by the user. Most of our simpler use cases already updated as expected, but some trickier cases with indirect relationships did not. For example, changing the date taken of a photo updated the data model for the photo, but deleting a photo did not necessarily ensure the photo was removed from all the cached albums, groups, and galleries to which it belonged. (Note that the photo was removed correctly from the backend, just not from the cached representation of those entities on the client.)</p> <p>Cleaning up these relationships using change events between models helped, but didn’t solve all our problems. When someone outside of the local session (read: another user) changed data, it would not reflect in the current session. The only way to catch changes from outside the current session was to be more aggressive about evicting models.</p> <h3>2. Nuclear option</h3> <div style="width:480px;"> <a href="https://www.flickr.com/photos/sdasmarchives/8091816484/">https://www.flickr.com/photos/sdasmarchives/8091816484/</a><br />   </div> <p>The pendulum swung all the way in the other direction — instead of surgical removal of data models we knew to be out-of-date, what would happen if we removed all cached data on every navigation? This prototype was quick to build, and incredibly destructive. By doing this, all our cached data always remained as fresh as could be, but we essentially reverted to Web 1.0 — with the exception of the Reboot framework, everything was reloaded on every page.</p> <p>Not surprisingly, this blew up API traffic (locally only! did not unleash that disaster at scale), and inflated page load times like a Jeff Koons sculpture. It did give us some baseline timing metrics we could point to as worst-case scenarios, however. The next step was to swing the pendulum back toward the middle — to a carefully-knitted solution that would preserve fast page loads and navigation, while ensuring the freshest data we could serve up.</p> <h3>3. Refetch on navigate</h3> <div style="width:340px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/daveemerson/4296539211/" title="fetched by Dave's Domain, on Flickr"><img decoding="async" loading="lazy" src="https://farm5.staticflickr.com/4029/4296539211_c054e6b762_z.jpg" width="498" height="640" alt="fetched"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>At this point, our challenge was to find a solution that would keep navigation fast, API traffic slim, and pick up all changes to session data, whether local or remote. We ended up with a solution we call “refetching”: evicting and requesting new data models as the model is needed by the application. But when?<br /> We could refetch periodically or on a user action; we determined that the best time to trigger a refetch was on navigation — when the user navigates, cached models become eligible for refetching. Specifically, when the user navigates between sections of the site, refetching is triggered. This proved to be the happiest medium between speed and freshness.</p> <p>A high-level outline of how the refetching strategy works:</p> <ul> <li>The user loads a page; data are requested from the API, and models are cached. As new models are created, they’re marked as being fresh.</li> <li>The user navigates to another site section (e.g. Photostream → Search); all freshness marks are removed from all models. They’re now all eligible for refetching.</li> <li>As Reboot builds the new page, it requests data models from the cache. Since they no longer have their seal of freshness, they are refetched, and marked as fresh once retrieved and cached.</li> </ul> <p>One important note — refetching is not triggered on browser back/forward navigation. Users expect near-immediate navigation, thanks to browser caching, when navigating to already-viewed content. Therefore, we refetch only when the user clicks a link to navigate to a new site section.</p> <h3>4. Miscellany</h3> <p>There were a couple other options we considered and rejected from the start, but they’re worth mentioning here.</p> <p>One was a <a href="https://en.wikipedia.org/wiki/Time_to_live">TTL (time-to-live) algorithm</a>, commonly used in caching applications. TTL algorithms expire data and evict from the cache a certain amount of time after they’re written or last updated. The arbitrary nature of TTL would mean that users would sometimes have fresh data and sometimes stale; it would be fresh more often than without any solution, but freshness would vary arbitrarily and would not result in much of an improvement on user experience.</p> <p>The other was to write an algorithm that tracks the amount of time since a data model was last accessed, and refetch when it grows too old. While this sounded interesting at first, it has the same flaw as a standard TTL algorithm — freshness becomes arbitrary. It’s also more complex to implement, and might end up not being worth the complexity.</p> <h2>The doing (implementation)</h2> <p>So that was it! Refetch on navigate, all done. Right?….of course not. With the general strategy in place, the devil started sneaking around in all the details. Some of the highlights:</p> <h3>Exemptions</h3> <p>It proved to be not the best idea to evict on all navigation. For example, in Reboot we often preload photo metadata models on pages with lists of photos, in order to make navigation into the photo page snappy. The refetch setup therefore has an exemption config that allows us to easily retain models when navigating into, away from, or between specific site sections.</p> <h3>Child models</h3> <p>We often have parent-child associations between data models. For example, the data model for a photo has a reference to a data model for the author of the photo. When the photo model is refetched, the person model must be refetched as well. This means the function doing the eviction and refetching has to recurse through all child models.</p> <h3>Collections</h3> <p>An issue similar to child models above, but more complex, is the case of a model containing a list of other models. For example, the data model for a person’s photostream contains a list of photo models.</p> <p>What made this particularly tricky is pagination and filtering — say you load the first 2 pages of your photostream, set your view filter to private, jump to page 5, switch the view to “Date taken”, and navigate away and back to your photostream…imagine the mess of different models with partially-loaded collections. Evicting one parent model, and its children, might evict photo models from the collection within another, without properly refetching. The solution here actually lay in the controller responsible for fetching pages: if a requested page of models is not already completely in-cache, a refetch will always happen to ensure we have all the data, in its freshest state.</p> <h3>Refetch only once per page view</h3> <p>Critical to the refetch-on-navigation strategy is to refetch only once per navigation. This was not too difficult, but essential to get right. We accomplish this by adding a flag when a model is initially fetched and upserted into the cache. When navigating to a new, non-exempt site section, all those flags are cleared, and any model requested by the new page will be refetched. When refetched, the model is again upserted into the cache and marked as fresh, until the next navigation.</p> <h2>But did it fresh?</h2> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/libellule24/4438085129/" title="Go on without me by Senorita Lena, on Flickr"><img decoding="async" loading="lazy" src="https://farm5.staticflickr.com/4019/4438085129_58cbbeb2f9_z.jpg" width="640" height="427" alt="Go on without me"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br />   </div> <p>With the thinking and the doing out of the way, it was time to push all this to production. Because these changes are essentially pulling the rug out from underneath the data layer on every navigation, we had to tread very carefully in order to prevent any negative impact to the end user experience.</p> <p>We did very thorough manual and automated testing across all of Reboot. We left the feature turned on for staff users for a while, to be able to respond to any bug reports. Finally, the time came to test on Real People. There were three things we needed to keep an eye on: errors (of course), impact on page navigation timing, and API traffic. Since refetching implies more requests for data, we needed to be sure that we were keeping the user experience smooth and fast, and also that we weren’t blowing up our data centers.</p> <div style="width:480px;">  <br /> <a href="https://www.flickr.com/photos/karolfranks/6296290871/">https://www.flickr.com/photos/karolfranks/6296290871/</a><br /> <em>All in</em><br />   </div> <p>In order to get a good read on these things, though, we had to go all in. Letting in just a small percentage of users would not give reliable numbers for timing or traffic impacts, due to the noise inherent in relatively small sample sizes. So, we did something unusual: we turned on refetching for all users for a short period of time. We flipped on refetching and kept an eagle eye on our stats for 2 hours, then reverted; then, we took a careful look at the aggregated data to see how the experiment went.</p> <p>Surprisingly, the impact on both timing and traffic was relatively low. After some thought, we decided this is most likely because the changes disproportionately impact people on long sessions, say a Flickr tab open for hours or days. Most people don’t hang around that long; they come, they go. Also, the photo page represents north of 90% of our page views, and is exempt from refetching (see Exemptions above).</p> <p>So where did we end up? A negligible bump in navigation timing and API traffic, and fresher data for all. Perhaps an anticlimactic resolution, but the story we’ve heard today outlines a serious consideration for anyone building an application with a data caching layer: keep in mind from the beginning how you plan to deal with stale data, but in a way that keeps all the other benefits of a single-page application.</p> <div style="width:480px;"> <a data-flickr-embed="true" href="https://www.flickr.com/photos/pinguino/6800265589/" title="#CCC is a breadcat by pinguino, on Flickr"><img decoding="async" loading="lazy" src="https://farm8.staticflickr.com/7151/6800265589_3204a4723b_z.jpg" width="640" height="427" alt="#CCC is a breadcat"></a><script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script><br /> <em>Busting through staleness. Yep.</em> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> <span class="sep"> | </span> <span class="tag-links"> <span class="entry-utility-prep entry-utility-prep-tag-links">Tagged</span> <a href="https://code.flickr.net/tag/api/" rel="tag">api</a>, <a href="https://code.flickr.net/tag/caching/" rel="tag">caching</a>, <a href="https://code.flickr.net/tag/data-model/" rel="tag">data model</a>, <a href="https://code.flickr.net/tag/eviction/" rel="tag">eviction</a>, <a href="https://code.flickr.net/tag/model/" rel="tag">model</a> </span> </footer> </article> <article id="post-3042" class="post-3042 post type-post status-publish format-standard hentry category-infrastructure category-performance"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/07/10/optimizing-caching-twemproxy-and-memcached-at-flickr/" rel="bookmark">Optimizing Caching: Twemproxy and Memcached at Flickr</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/07/10/optimizing-caching-twemproxy-and-memcached-at-flickr/" title="7:59 pm" rel="bookmark"><time class="entry-date" datetime="2015-07-10T19:59:40-07:00">July 10, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/tague/" title="View all posts by Tague Griffith" rel="author">Tague Griffith</a></span></span> </div> </header> <div class="entry-content"> <h2><b>Introduction</b></h2> <p>Flickr places a high priority on our users’ experience, and a critical part of that experience is the speed of the interface. Regardless of the client you use to access Flickr, caching the proper data and the speed at which our servers can access cached data is critical to delivering on that quality user experience. The more effective our caching strategy is, the better the Flickr experience will be for our users. This is true for all the layers of caching we deploy at Flickr from the photo caches to the process data caches buried deep in the system. <a href="http://code.flickr.net/2014/08/26/performance-improvements-for-photo-serving">In a previous post</a>, we looked at how regional photo caching improved photo serving time in other countries, in this post we’re going to dive down into the innards of Flickr’s software stack and take a look at how we improved <a href="http://memcached.org/" target="_blank">Memcached</a> performance for our backend systems.</p> <p>Back in the olden days (pre-2014), we accessed our Memcached systems through a mix of direct reads from our web servers and writes through a Flickr-developed proxy. Our proprietary proxy system, Cerberus, handled a whole host of responsibilities. In addition to Memcached set operations, Cerberus managed database updates, the bulk of our Redis accesses, cache consistency<img decoding="async" loading="lazy" class=" size-medium wp-image-3046 alignright" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/06/cerberus-based-memcached-architecture.png?w=300" alt="Cerberus Based Memcached Architecture" width="300" height="174" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cerberus-based-memcached-architecture.png 592w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cerberus-based-memcached-architecture.png?resize=150,87 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cerberus-based-memcached-architecture.png?resize=500,290 500w" sizes="(max-width: 300px) 100vw, 300px" /> (which is why we directed our writes through Cerberus), and a few other miscellaneous transactions. As Flickr’s traffic and functionality had grown, Memcached set operation performance wasn’t keeping up, so we needed to consider how to address the gap.</p> <p>Since the development of Cerberus, the software landscape had drastically changed. When we developed Cerberus, there was no comparable software available, but now several open source projects exist that provide similar proxy services. On top of the availability of open source tools, Flickr’s traffic and usage patterns have changed over the course of a decade, changing the requirements we had for a proxy system. Needless to say we had a lot of questions to ask ourselves before we dived into revising the caching architecture.</p> <p>After years of operation, we had a good picture into the strengths and weaknesses of our current architecture, so when we started thinking about revising it, a few lessons from the past years really stood out. One thing that we learned over the years was that Cerberus’ lack of a single purpose made it difficult to troubleshoot operational issues with Memcached. Due to the lack of isolation, a downstream issue with a user database, could impact Memcached access time. Whatever came next had to isolate cache requests from other data accesses.</p> <p>Experience had shown us that Memcached get operation latency was a key performance metric, and we learned through trial and error that placing our current Cerberus proxy between the web servers and the Memcached hosts added more network latency than we were willing to tolerate. Unfortunately, our options for connection pooling and more efficient use of connections were limited in PHP, so we had little recourse than to suffer with a high connection load and fluctuating connections against our Memcached servers. The next generation system would have to carefully monitor get operation timings and ensure we didn’t introduce more latency into the process.</p> <p>So as 2014 rolled around, we started to look into an alternative to Cerberus for accessing the Memcached systems. Should we build a Cerberus 2.0? Should we look at an open source alternative? As we weighed our options, one alternative that stood out because it was quite successful in other parts of Yahoo and throughout the industry was Twitter’s Twemproxy.</p> <h2><b>Introducing Twemproxy</b></h2> <p><a href="https://github.com/twitter/twemproxy" target="_blank">Twemproxy</a> is a lightweight daemon that proxies requests to a pool of Memcached instances. It provides the following features that we believed would improve our caching infrastructure:</p> <ul> <li>Consistent hashing: Figuring out what hosts in the ring on which to store and retrieve cache data. While we had implemented consistent hashing before Twemproxy, it was previously left to the client libraries to sort out.</li> <li>Connection pooling: Reuse of network connections to the Memcached instances, cutting down significantly on the connection load to the Memcached daemons.</li> <li>Command Pipelining: Accumulating requests destined for the same host and sending as a combined payload. This feature further reduces connection load and network overhead to the cache processes.</li> </ul> <h2><b>Resulting Architecture</b></h2> <p>We implemented a solution where each web server host had two local Twemproxies, forwarding to the Memcached rings.</p> <p><a href="https://wp.flickr.net/wp-content/uploads/sites/3/2015/06/twemproxy-based-memcached-architecture.png"><img decoding="async" loading="lazy" class=" size-medium wp-image-3047 aligncenter" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/06/twemproxy-based-memcached-architecture.png?w=300" alt="Twemproxy Based Memcached Architecture" width="300" height="146" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/06/twemproxy-based-memcached-architecture.png 596w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/twemproxy-based-memcached-architecture.png?resize=150,73 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/twemproxy-based-memcached-architecture.png?resize=500,244 500w" sizes="(max-width: 300px) 100vw, 300px" /></a></p> <p>In the resulting architecture, all Memcached operations go through twemproxy. The change accomplished many goals, including:</p> <ul> <li>Providing a dedicated system for Memcached requests that was isolated from other systems</li> <li>Reducing the connection load on our Memcached servers through Twemproxy’s connection pooling. We experienced a 75% overall reduction of TCP connections to Memcached nodes</li> <li>Improved overall caching latency. This was a benefit that we didn’t necessarily expect. With Twemproxy, we found that get operations had a 5% reduction in mean processing time and set operations had a 40% in mean processing time</li> </ul> <h2><b>The Road to Twemproxy</b></h2> <p>As nice as it would be to say we dropped in Twemproxy, declared victory and went for ice cream, we still had to solve a few interesting challenges along the way: maintaining availability, dealing with disparate consistent hashing schemes, and re-implementing cache coherency.</p> <h3><b>If One is Good, Two is Even Better</b></h3> <p>From the start, we recognized that the simple daemon model of Twemproxy would need to be managed carefully. Each deploy through our continuous deployment system could result in changes to the Memcached hosts configuration. And unfortunately, configuration changes for Twemproxy require the process to be restarted to take effect. We measured the time to restart Twemproxy in the 1-2 second range, but even for these necessary restarts, it was too much of an interruption for the clients.</p> <p>Our solution was to run two instances of the daemon on every host that needed the service and manage a careful synchronization between the restarts. This restart “dance” was wired into the process that deploys the configuration changes to all the Memcached clients. A couple of patches have been proposed to allow configuring Twemproxy without restarting it, but none of them have yet made the master branch.</p> <h3><b>When is Ketama not Ketama?</b></h3> <p><a href="http://www.audioscrobbler.net/development/ketama/" target="_blank">Ketama</a> is a popular consistent hashing algorithm used by many systems to determine where to place a particular key in a multi-node caching system. Out of the box, Twemproxy uses an implementation of the Ketama algorithm that is compatible with libketama, the C library which is the most commonly used implementation of the Ketama algorithm.</p> <p>Our initial implementation of consistent hashing was done using Ketama, but with the <a href="https://code.google.com/p/spymemcached/">Spymemcached</a> Java library. It turns out that Spymemcached has a slight variation in the implementation that makes it incompatible with Twemproxy.</p> <p>Our transition from our current system to Twemproxy had to happen live, and a sudden change in the cache algorithm would have a painful (and unacceptable) impact on our database systems. How could we get across this bridge? Ultimately, we had to patch Twemproxy’s implementation of Ketama to match Spymemcached to maintain a consistent implementation of the Ketama algorithm.</p> <h3><b>Redis latency in propagating cache clears</b></h3> <p>Until we figure out how to change the speed of light, the only way we are going to make Flickr fast across the world, is through multiple data centers conveniently located near our users. While this is way easier than changing the speed of light, it’s not without its complications.</p> <p><a href="https://www.flickr.com/photos/arthur-caranta/2925352521/" data-flickr-embed="true"><img decoding="async" class="flickr-embed" src="https://farm4.staticflickr.com/3295/2925352521_517c291633_z.jpg" alt="What do they compute at Night ?" /></a></p> <p>Caches between the data centers have to be kept consistent. Some caches, like photo caches, deal with immutable data and are easy to keep in synch, others like Memcached systems have read-write data which is harder. Our approach to handling cache consistency in our Memcached systems was to invalidate stale keys in other colo facilities whenever a process updated a value. As we mentioned previously, Memcache write operations were directly through our Cerberus proxy specifically so Cerberus could dispatch a cache invalidation event to other colo facilities. The migration to Twemproxy would not be complete, until we implemented a new solution for cache invalidation.</p> <p>In our Twemproxy-based architecture, we decided to take the responsibility for cache invalidation our of the hands of the data proxy and push it into the client libraries we used to access Memcached. Whenever a client updates a Memcache key, it enqueues a corresponding cache invalidation event into a distributed Redis queue. We then deployed simple, single-purpose Java daemons to process the cache invalidation events from the Redis queue and delete the corresponding keys in their local Memcached systems. A diagram of the system, appears below: <img decoding="async" loading="lazy" class=" size-large wp-image-3079 aligncenter" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?w=660" alt="Cache Clear - Blog Post" width="660" height="339" srcset="https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png 1092w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?resize=150,77 150w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?resize=800,411 800w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?resize=768,395 768w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?resize=1024,526 1024w, https://code.flickr.net/wp-content/uploads/sites/3/2015/06/cache-clear-blog-post.png?resize=500,257 500w" sizes="(max-width: 660px) 100vw, 660px" /></p> <p>The wrinkle with this approach was that the enqueuing of clear keys would occasionally take 20 times longer than the normal mean time, pushing cache sets up to 40ms. After much digging, we found that the spikes were happening when the clearing daemons dequeued a batch of keys. The dequeuing daemons were performing operations across a WAN. Due to the single-threaded nature of Redis, it would periodically block the queue for adding keys for 10s of milliseconds. Once we figured that out, the fix was a matter of keeping separate in- and out-queues, and moving the keys from in to out with a <i>local</i> daemon, which significantly reduced the blocked time for writing keys.</p> <h2><b>Conclusion</b></h2> <p>Caching is crucial to a high-traffic site like Flickr, and we have taken a big stride in making our Memcached utilization more effective. Using Twemproxy, we were able to clean up an internal system, reduce the connection load on our caching daemons, and even make modest improvements to caching latency for all clients. Although we faced some technical challenges in implementing twemproxy for Memcached, particularly in propagating cache clear events, it was ultimately well worth the engineering investment. After several months, our implementation of Twemproxy has proven to make a positive contribution to caching speed and ultimately the experience of a responsive site for our users.</p> <p>If you dream in low latency and love to rip that extra 10 microseconds of overhead out of an operation, we’d love to have you! Stop by our <a href="http://www.flickr.com/jobs">Jobs page</a> and tell us how awesome you are.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/infrastructure/" rel="category tag">infrastructure</a>, <a href="https://code.flickr.net/category/performance/" rel="category tag">performance</a> </span> </footer> </article> <article id="post-3053" class="post-3053 post type-post status-publish format-standard hentry category-uncategorized"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/06/25/real-time-resizing-of-flickr-images-using-gpus/" rel="bookmark">Real-time Resizing of Flickr Images Using GPUs</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/06/25/real-time-resizing-of-flickr-images-using-gpus/" title="2:15 am" rel="bookmark"><time class="entry-date" datetime="2015-06-25T02:15:46-07:00">June 25, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/archieflickr/" title="View all posts by Archie Russell" rel="author">Archie Russell</a></span></span> </div> </header> <div class="entry-content"> <p>At Flickr we work with a huge number of photos. Our users upload over 27 million photos a day, and our total collection has over 12 billion photos. This is fantastic! As usage grows, we are always looking for ways to use our storage more efficiently. Recently our storage team wrote about some <a href="http://yahooeng.tumblr.com/post/116391291701/yahoo-cloud-object-store-object-storage-at">new commodity storage technology</a> now in use at Flickr which increases efficiency. But we also looked into how much data we store for each photo. In the past we stored many sizes of every photo to make serving fast. We wanted to challenge that model and find the minimal set of data to store.</p> <h2>Thumbnail Footprint Reduction</h2> <p>One of our biggest opportunities for byte per photo improvement is through reduction in the footprint of Flickr’s “thumbnails”. Thumbnail is a bit of a misnomer at Flickr; our thumbnails are as large as 2048 pixels on their longest side, so at Flickr we usually refer to these as <i>resizes</i>. We create these resizes in order to provide a consistent, fast experience for our users over a variety of use cases.</p> <p class="figure"><img decoding="async" loading="lazy" src="https://farm1.staticflickr.com/324/18942644270_e4c1779d8b_o.png" alt="" width="800" height="600" /><br /> <span class="caption"><br /> Different sizes used in different contexts. From left to right: Cameraroll uses small thumbnails, to enable fast navigation through many sizes. Our Photo Page uses our largest, most detailed sizes. Search uses sizes in between these two extremes. Red panda photos by <a href="https://www.flickr.com/photos/mathiasappel/">Mathias Appel</a>.<br /> </span></p> <p>The selection of sizes has grown semi-organically over the years, and all told, we serve eleven different resizes per photo which, in sum, use nearly as much storage as the original photo. Almost 90% of this storage is held in the handful of resizes 640px and larger, so we targeted our efforts at eliminating some of these sizes.</p> <p class="figure"><img decoding="async" loading="lazy" src="https://farm1.staticflickr.com/522/19133157391_b0e1c51fa9_o.png" alt="" width="800" height="600" /><br /> <span class="caption"><br /> Left: Distribution of byte size by resize dimension. Storage is concentrated in images with largest dimensions. Right: size distribution after largest sizes eliminated.<br /> </span></p> <h2>A Few Approaches</h2> <p>A simple approach to this problem would be just to cease offering some of the larger sizes. For instance, we could drop the 1600px image from our API and require the design to adjust. However, this requires compromises that we didn’t want to take on. Instead we took on a pretty ambitious goal: maintain our largest resize, usually 2048px wide, as a source image and create any other moderate or large-sized resizes on-the-fly from this source, without sacrificing image quality or significantly affecting performance. Using the original uploaded photo as a resize source image was impractical, as these can be very large and exist in a variety of formats.</p> <p>Sounds easy, right? We already resize images when users upload, so why not just use that same technology on serving. Well, almost. The problem with the naive approach is that high-quality resizing of JPEGs is a lot slower than is widely known. A tool we use frequently, <a href="http://www.graphicsmagick.org/">GraphicsMagick</a>, produces beautiful images but takes over 225ms to resize a 2048px JPEG down to 1600px, depending on quality settings. This is slow enough that this method would impact user experience, and would require many CPUs to handle our load. <a href="https://github.com/yahoo/ygloo-ymagine">Ymagine</a>, a high-performance CPU-based tool we’ve open sourced, is twice as fast as GraphicsMagick(!). We use Ymagine extensively on smaller images, but for the large sizes we’re targeting we needed even more performance. A GPU-based solution ultimately filled our needs.</p> <h2>Our GPU-based Solution</h2> <p>We created a tier of dedicated <i>resize servers</i>, each with an GPU co-processor. Each of these boards has two GPUs, each with 1500+ “cores”, running at just under 1GHz. These cores aren’t anywhere near as performant as a CPU core, but there are many of them. We tested a range of server-grade boards to find the best performing type for our workload. Many manufacturers offer consumer-grade boards with incredible specifications and lower price points, but these lack server-grade cooling and other features such as ECC RAM. One member of our team had experience using these lower grade boards in a previous application and recommended against it.</p> <p class="figure"><img decoding="async" loading="lazy" src="https://farm1.staticflickr.com/297/19130003555_8a1896d66e_o.png" alt="" width="800" height="600" /><br /> <span class="caption"><br /> Resize system architecture<br /> </span></p> <p>On these resize servers we run a fairly vanilla Apache with a plugin written in C++. This server responds to resize requests, reads our source image from disk into shared memory, and hands off requests off to persistent <i>resize daemons</i> that do all communication with our GPUs. A daemon-type approach is necessary due to a somewhat lengthy initialization process with our GPUs.</p> <p>Our resize daemons transfer JPEGs from shared memory to GPU device memory. Once here, the real image processing takes place. The JPEGs are decoded, cropped, sharpened, resized, re-sharpened as needed, re-encoded as JPEGs, and finally transferred back to shared memory. From shared memory, our Apache module returns the resized JPEG to the caller.</p> <p class="figure"><img decoding="async" loading="lazy" src="https://farm1.staticflickr.com/451/18506807884_e9cb187c49_o.png" alt="" width="800" height="600" /><br /> <span class="caption"><br /> A simple resize pipeline. Post-sharpening overcomes fuzziness introduced when downscaling.<br /> </span></p> <p>There are several accepted resize algorithms, but to retain the Flickr “look”, we implemented the same Lanczos resize and kernel sharpening algorithms that we’ve used for years in CUDA. This had the added benefit of being able to directly compare images generated through GraphicsMagick and our GPU-based code.</p> <h2>Performance</h2> <p>With significant optimization, this code is able to resize our 2048px JPEGs to 1600px in under 16ms. This is more than 15x faster than GraphicsMagick and nearly 10x faster than Ymagine. Resizes from 2048px to 640px take under 10ms. Equally noteworthy, at peak load, each resize server can perform over 300 resizes per second.</p> <p class="figure"><img decoding="async" loading="lazy" src="https://farm1.staticflickr.com/262/18509003323_f8c1fa64c0_o.png" alt="" width="800" height="600" /><br /> <span class="caption"><br /> Performance of different resize approaches.<br /> </span></p> <p>Although these timings are quite fast, the source image for our resizes is larger, byte-wise, than the images it is resizing to, requiring additional I/O. For example, a typical 2048px source JPEG is roughly 600kB and our typical 1024px JPEGs are just under 200kB. This difference in size leads to roughly 35ms additional I/O time per resize.</p> <h2>Taking it slow</h2> <p>As our GPU code is new and images are our most important product, this change carries some risk. We’ve addressed this with extensive testing, progressive rollout and provisions for rollback. We also used some insights into our user behavior to roll this solution out in a very controlled manner.</p> <h2>Conclusion</h2> <p>This system is currently in production and as we roll it out more fully, has the potential to cut the resize footprint of the majority of our photos by 50%, with negligible impact on performance and image appearance. We also have the ability to apply this same footprint reduction technique to images uploaded in the past, which has the potential to reduce our storage growth to zero for a significant period of time.</p> <h2>Credits</h2> <p>This project would not have been possible with hard work of Peter Norby, Tague Griffith, John Ko and many others.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/uncategorized/" rel="category tag">Uncategorized</a> </span> </footer> </article> <article id="post-2887" class="post-2887 post type-post status-publish format-standard hentry category-photos category-search"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/03/24/much-photos/" rel="bookmark">Much Photos!</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/03/24/much-photos/" title="4:54 pm" rel="bookmark"><time class="entry-date" datetime="2015-03-24T16:54:45-07:00">March 24, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/ericsoco/" title="View all posts by ericsoco" rel="author">ericsoco</a></span></span> </div> </header> <div class="entry-content"> <h2>Introducing the New! Shiny! Photolist framework</h2> <p class="figure"> <img decoding="async" src="https://wp.flickr.net/wp-content/uploads/sites/3/2015/01/photolist-sky.jpg" width="800" alt="photolist-sky"><br /> <span class="caption">Blue skies. Mostly.</span> </p> <p>Here at Flickr, we have photos. Lots of photos. Like, <a href="https://www.flickr.com/photos/8500690@N02/11155239475/in/set-72157638249604334">billions and billions</a> of photos. So, it’s pretty important for us to be able to show you more than one at once.</p> <p>We have used what we call the “justified algorithm” to lay out photos <a href="http://code.flickr.net/2013/06/">for a while now</a>, but as we move more and more pages onto our new-ish <a href="http://isomorphic.net/javascript">isomorphic</a> <a href="http://nodejs.org">node.js</a> stack, we determined it was time to revisit the algorithm and create an updated implementation.</p> <p>A few of us here in <a href="https://www.flickr.com/jobs/frontend_engineer/">Frontend-landia</a> got together to figure out all the things this new shiny should be able to do. With a lot of projects in full swing and on the near horizon, we came up with a pretty significant list, including but not limited to:<br /> – Easy for developers to use<br /> – Fit into any kind of container<br /> – Support pagination (in both directions!) and infinite scroll<br /> – Jank-free, butter-silky-baby-smooth scrolling<br /> – Support layouts other than justified, like square thumbnails and grid layout with native aspect ratio</p> <p>After some brainstorming, drawing of diagrams, and gummi bear consumption, we got to work building out the framework and the underlying algorithm.</p> <h2>Rejustification</h2> <p class="figure"> <a title="Photolist sketches by eric socolofsky, on Flickr" href="https://www.flickr.com/photos/ericsoco/16281033181"><img decoding="async" src="https://farm9.staticflickr.com/8566/16281033181_73bf58ac84_h.jpg" width="800" alt="photolist-sky"></a><br /> <span class="caption">Drawing of diagrams</span> </p> <p>The basics of the justified algorithm aren’t too complex. The goal is for the layout module to accept a list of photo aspect ratios, and return a list of rectangles. A layout consists of a number of rows of items (photos), each with a target height and allowable height deviation above and below. This, along with the container width, gives us a minimum and maximum row aspect ratio.</p> <p class="figure"> <a title="Photolist: variable row height by eric socolofsky, on Flickr" href="https://www.flickr.com/photos/ericsoco/16096960347"><img decoding="async" src="https://farm9.staticflickr.com/8579/16096960347_1e90da9a04_b.jpg" width="800" alt="Photolist: variable row height"></a><br /> <span class="caption">Fig. 1: The justified algorithm: dimensions</span> </p> <p>We push each photo into a row; once the row is filled up, we move on to the next. It goes a little something like this:</p> <ol> <li>Iterate over each photo in the list to display</li> <li>Create a new row if there’s not currently an open row</li> <li>Attempt to add the photo to the current row at its native aspect ratio and at the target row height</li> <li>If the new row aspect ratio is less than the minimum row aspect ratio, continue adding photos until the aspect ratio is greater than the maximum aspect ratio</li> <li>Either keep or drop the last added photo, depending on which generates a row aspect ratio closer to the target row aspect ratio; adjust the row height as needed, and seal the row</li> <li>Repeat until all the photos have been laid out.</li> </ol> <p class="figure"> <a title="Photolist: row filling by eric socolofsky, on Flickr" href="https://www.flickr.com/photos/ericsoco/15660403594"><img decoding="async" src="https://farm9.staticflickr.com/8589/15660403594_cc6d380659_b.jpg" width="800" alt="Photolist: row filling"></a><br /> <span class="caption">Fig. 2: The justified algorithm: row filling</span> </p> <h2>It’s Never That Easy…</h2> <p>The justified algorithm described above is the primary responsibility of the layout module. In practice, however, there are a number of other things the layout must handle to get good results for all use cases, and to communicate the results to other parts of the framework.</p> <h3><a style="text-decoration:none;color:black;" name="diffs"></a>Diffs</h3> <p>One key feature of the layout module is how it organizes its results. To minimize the amount of processing required to update photos as the layout changes, the layout module returns pre-sorted diffs, each with a specific purpose:</p> <ul> <li>new items, used to create new photos and put them in place</li> <li>layout-changed items, used to resize/reposition existing photos</li> <li>visibility-changed items, used to wake/sleep existing photos</li> <li>widows and orphans (leading and trailing items) (read on!)</li> </ul> <p>The container view can process only the parts of the layout response that are necessary, given the current state of the whole framework, to keep processing time down and keep performance up.</p> <h3>Widows and orphans</h3> <p class="flickr-photo"> <a title="Annie the Musical, by Eva Rinaldi, on Flickr" href="https://www.flickr.com/photos/evarinaldiphotography/6626178677"><br /> <img decoding="async" src="https://farm8.staticflickr.com/7021/6626178677_9bf0bb25d5_z.jpg" alt="Annie the Musical," style="height:400px;"><br /> </a><span class="caption"><a href="https://www.flickr.com/photos/evarinaldiphotography/6626178677">Annie the Musical,</a> by <a href="https://www.flickr.com/photos/evarinaldiphotography/">Eva Rinaldi</a></span> </p> <p>Some photolist pages on Flickr use infinite scrolling, and some display results one page at a time. Regardless of how a page shows its photos, it starts to feel messy when there is an incomplete row of photos hanging off the end of the page. If there is more content in the set, the last row should be full. However, since we fetch photos from the API in fixed batch sizes, things don’t always work out so nicely, leaving “leftovers” in the bottom row. Borrowing from <a href="http://en.wikipedia.org/wiki/Widows_and_orphans">typesetting terminology</a>, we call these leftover photos orphans. (We can also paginate backwards; leftovers at the top are technically widows but we’ll just keep using the term orphans for simplicity.)</p> <p>The layout notes these incomplete rows and hides them from the rest of the framework until the next page of content loads in. (This led to frequent and questionable metaphors about “orphan suppression” and “orphan rehydration.”) When orphans are to be hidden, the layout simply keeps them out of the diff. When the orphans are brought back in as the next page loads, the layout prepends them to the next diff. The container view is none the wiser.</p> <p>This logic gets even more fun when you consider that it must perform in all of these use cases:</p> <ul> <li>fixed page size (book-style) pagination</li> <li>downward-scrolling infinite pagination</li> <li>upward-scrolling infinite pagination (enter into an “infinite” content set somewhere other than the beginning); this requires right-to-left layout!</li> </ul> <p>There’s also the case of the end of an “infinite” content set (scrolling down to the end or up to the beginning); in these cases, we still want the row to appear complete, and still must maintain the native aspect ratio of the photo. Therefore, we allow the row height to grow as much as it needs in this case only.</p> <h3>Bonus Round!</h3> <p>You might have noticed that Flickr is kind of a big site with, like, lots of photos. And we display photos in lots of different ways, with lots of different use cases. The photolist framework bends over backwards to support all of those, including:</p> <ul> <li>forward and backward pagination</li> <li>infinite scroll and fixed page-size pagination</li> <li>specific aspect ratios (e.g. squares)</li> <li>fixed number of rows</li> <li>fast relayout (only a few milliseconds for thousands of photos)</li> </ul> <p>Going into detail on each of those features is way beyond the scope of this blog post, but suffice it to say the framework is built to handle just about anything Flickr can throw at it. The one exception is the upcoming Camera Roll (coming soon to those of you who don’t yet have it!), which is Too Extreme for this framework, so we devised something special just for that page.</p> <h2>The whole enchilada</h2> <p class="flickr-photo"> <a title="Mmm... enchiladas by jeffreyw, on Flickr" href="https://www.flickr.com/photos/jeffreyww/8221489760"><br /> <img decoding="async" src="https://farm9.staticflickr.com/8064/8221489760_7062cf85b0_k.jpg" alt="Mmm... enchiladas," width="800"><br /> </a><span class="caption"><a href="https://www.flickr.com/photos/jeffreyww/8221489760">Mmm… enchiladas,</a> by <a href="https://www.flickr.com/photos/jeffreyww/">jeffreyww</a></span> </p> <p>The layout is at the heart of the photolist framework, but wait — there’s more! The main components of the framework are the layout (dissected above), the container view / controller, and the subviews (usually containing photos).</p> <p class="figure"> <a title="Photolist: the whole enchilada by eric socolofsky, on Flickr" href="https://www.flickr.com/photos/ericsoco/16281062391"><img decoding="async" src="https://farm8.staticflickr.com/7520/16281062391_60980d9764_b.jpg" width="800" alt="Photolist: the whole enchilada"></a><br /> <span class="caption">Fig. 3: Relationship of view/controller, layout, and subviews, and changing subview states during downward scroll</span> </p> <p>The container view does a lot of fancy things, like:</p> <ul> <li>loading in photos as you scroll down or paginate</li> <li>triggering a relayout when the container size changes (i.e. when you resize your window)</li> <li>matching up server-rendered HTML with clientside JavaScript objects (see <a href="http://isomorphic.net/javascript">isomorphic JavaScript</a>, and an upcoming blog post about the Hermes stack at Flickr).</li> </ul> <p>Its primary job, though, is to act as the conduit between the layout module and the individual subviews.</p> <p>Every time a layout is processed or changes, it returns a “layout response” to the container view. The layout response contains a list of rectangles and wake/sleep flags (actually, a list of lists; see <a href="#diffs">Diffs</a> above); the container view relays that new information on to each individual subview to determine position and visibility. The container view doesn’t even need to know about the layout details — each subview adjusts itself to its layout data all on its own.</p> <p>The subviews each have a decent amount of intelligence of their own, performing such tasks as:</p> <ul> <li>choosing the most appropriate photo file size to fit the layout rectangle</li> <li>adding/removing itself to/from the DOM as instructed by the layout to maintain good scroll performance</li> <li>providing an annotation and interaction layer for titles, faves, comments, etc.</li> </ul> <h2>Coming soon to a webpage near you</h2> <p>The new photolist framework is certainly not a one-size-fits-all solution; it’s tailored for Flickr’s specific use cases. However, we tried to design and build it to be as broadly useful for Flickr as possible; as we continue to move parts of the site onto the new frontend stack and innovate new features, it’s critical to have solid components upon which we can build Flickr’s future. The layout algorithm is probably useful for many applications though, and we hope you gained some insight into how you might implement your own.</p> <p>The photolist framework is already live in a number of places on the site, including the <a href="https://www.flickr.com/search?text=kittens">new Unified Search</a> pages (<a href="https://www.flickr.com/help/forum/en-us/72157651248603362/">currently in Beta</a>), the <a href="https://www.flickr.com/create">Create</a> / <a href="https://www.flickr.com/create/curatedcollection">Wall art</a> pages, the <a href="https://www.flickr.com/groups/flickrfriday">Group pool preview</a>, and is coming soon to a number of other pages.</p> <p>As always, if you’re interested in helping with that “and more” part, we’d love to have you! Stop by our <a href="https://flickr.com/jobs">jobs page</a> and drop us a line.</p> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/photos/" rel="category tag">photos</a>, <a href="https://code.flickr.net/category/search/" rel="category tag">search</a> </span> </footer> </article> <article id="post-2917" class="post-2917 post type-post status-publish format-standard hentry category-metrics"> <header class="entry-header"> <h1 class="entry-title"><a href="https://code.flickr.net/2015/03/04/browsers-in-2014/" rel="bookmark">33 Browser Stats You Just Might Believe</a></h1> <div class="entry-meta"> <span class="sep">Posted on </span><a href="https://code.flickr.net/2015/03/04/browsers-in-2014/" title="5:47 pm" rel="bookmark"><time class="entry-date" datetime="2015-03-04T17:47:15-08:00">March 4, 2015</time></a><span class="by-author"> <span class="sep"> by </span> <span class="author vcard"><a class="url fn n" href="https://code.flickr.net/author/pdokas/" title="View all posts by Phil Dokas" rel="author">Phil Dokas</a></span></span> </div> </header> <div class="entry-content"> <p>We care an awful lot about the kinds of browsers and computers visiting Flickr. As people update to the latest versions of their browsers, the capabilities we can build against improve, which lets us build cool new things. At the same time, if lots of people continue using older browsers then we have to do extra work to gracefully support them.</p> <p>These days we not only have incredibly capable browsers, but thanks to the transparent and rapid update process of Chrome, Firefox, and <a href="http://blogs.msdn.com/b/ie/archive/2015/01/22/project-spartan-and-the-windows-10-january-preview-build.aspx">soon Internet Explorer</a> (hooray!), we can rely on new features rapidly showing up en masse. This is crazy great, but it doesn’t mean that we can stop paying attention to our usage statistics. In fact, as people spend more time on their phones, there’s as much of a need for a watchful eye as ever.</p> <p>We’ve never really shared our internal numbers, but we thought it would be interesting to take a look at the browsers Flickr visitors used in 2014. We use these numbers constantly to inform our project planning. Since limitations in older browsers take time to support we have to be judicious in picking which battles to fight. As you’ll see below, these numbers can be quite dynamic with a popular browser dropping to nearly 0% market-share in just a year. Let’s dive in and see some specifics.</p> <p class="flickr-photo"><a href="https://www.flickr.com/photos/k-8/237296621" title="Fort Vancouver by Kate Dickerson, on Flickr"><img decoding="async" src="https://farm1.staticflickr.com/86/237296621_76c0b450d8_b.jpg" width="800" alt="Fort Vancouver"></a><br /> <span class="caption"><a href="https://www.flickr.com/photos/k-8/237296621">Fort Vancouver</a> by <a href="https://www.flickr.com/photos/k-8/">Kate Dickerson</a></span></p> <h2>Top level OSes and browsers</h2> <p>At the highest level we learn a lot by looking at our OS family data. Probably the most notable thing here is how much of our traffic is coming from mobile devices. Moreover, the rate of growth is eye-popping. And this is just our website – this data doesn’t include our iOS or Android clients at all. <strong>A quarter of our traffic is from mobile devices.</strong></p> <table class="data"> <caption>OSes in use on Flickr.com</caption> <thead> <tr> <th scope="col"></th> <th scope="col">2013 Q4</th> <th scope="col">2014 Q4</th> <th scope="col">Y/Y</th> </tr> </thead> <tbody> <tr class="odd"> <th scope="row">Windows</th> <td>56.55%</td> <td>50.61%</td> <td style="color:red;">-5.94</td> </tr> <tr class="even"> <th scope="row">Macintosh</th> <td>21.49%</td> <td>21.42%</td> <td style="color:red;">-0.07</td> </tr> <tr class="odd"> <th scope="row">iOS</th> <td>11.09%</td> <td>17.61%</td> <td style="color:green;">6.52</td> </tr> <tr class="even"> <th scope="row">Android</th> <td>5.39%</td> <td>7.82%</td> <td style="color:green;">2.43</td> </tr> <tr class="odd"> <th scope="row">Other</th> <td>5.48%</td> <td>2.54%</td> <td style="color:red;">-2.94</td> </tr> </tbody> </table> <p>Let’s slice things slightly differently and look at browser families. We greatly differ from internet-wide traffic in that IE isn’t the outright majority browser. In fact, it clocks in at only the #4 position. <strong>More than half of Flickr visitors use a Webkit/Webkit-heritage browser</strong> (Safari and Chrome, respectively). Chrome rapidly climbed into its leadership position over the last few years and it’s stabilized there. Safari is hugely buoyed by iOS’s incredible growth numbers, while IE has been punished by Windows’s Flickr market-share decline.</p> <table class="data"> <caption>Browsers in use on Flickr.com</caption> <thead> <tr> <th scope="col"></th> <th scope="col">2013 Q4</th> <th scope="col">2014 Q4</th> <th scope="col">Y/Y</th> </tr> </thead> <tbody> <tr class="odd"> <th scope="row">Chrome</th> <td>35.71%</td> <td>35.42%</td> <td style="color:red;">-0.29</td> </tr> <tr class="even"> <th scope="row">Safari</th> <td>24.11%</td> <td>27.50%</td> <td style="color:green;">3.39</td> </tr> <tr class="odd"> <th scope="row">Firefox</th> <td>17.94%</td> <td>18.29%</td> <td style="color:green;">0.35</td> </tr> <tr class="even"> <th scope="row">Internet Explorer</th> <td>13.98%</td> <td>10.31%</td> <td style="color:red;">-3.67</td> </tr> <tr class="odd"> <th scope="row">Other</th> <td>8.26%</td> <td>8.48%</td> <td style="color:green;">0.22</td> </tr> </tbody> </table> <h2>Fine-grained details</h2> <p>We can go a step further and see many details in the individual versions of OSes and browsers out there. It’s one thing to say “Windows is down 6% over the year” but another to say “the growth rate for the latest version of Windows is 350% year over year.” When we look at the individual versions we can infer quite a bit of detail around update rates and changes in the landscape.</p> <h3>OS version details</h3> <p>A few highlights:</p> <ul> <li>Windows 7 is on the decline, XP and Vista fell by roughly 50% each, and Windows 8 and 8.1 are surging ahead.</li> <li>iOS 8.1 and Android 5.0 don’t appear in the list due to their late appearance in Q4. Our current monthly numbers have iOS 8.1 far outpacing every other iOS version.</li> <li>OS X 10.10 has accelerated Mac user upgrades; since its launch 10.9 has shed over a percent per month, and the legacy versions have sharply accelerated their decline.</li> </ul> <table class="data"> <caption>OS versions in use on Flickr.com</caption> <thead> <tr> <th scope="col"></th> <th scope="col">2013 Q4</th> <th scope="col">2014 Q4</th> <th scope="col">Y/Y</th> </tr> </thead> <tbody> <tr class="odd"> <th scope="row">Windows NT</th> <td>3.39%</td> <td>0%</td> <td style="color:red;">-3.39</td> </tr> <tr class="even"> <th scope="row">Windows XP</th> <td>10.12%</td> <td>4.49%</td> <td style="color:red;">-5.63</td> </tr> <tr class="odd"> <th scope="row">Windows Vista</th> <td>3.56%</td> <td>2.41%</td> <td style="color:red;">-1.15</td> </tr> <tr class="even"> <th scope="row">Windows 7</th> <td>36.29%</td> <td>33.14%</td> <td style="color:red;">-3.15</td> </tr> <tr class="odd"> <th scope="row">Windows 8</th> <td>2.01%</td> <td>2.31%</td> <td style="color:green;">0.30</td> </tr> <tr class="even"> <th scope="row">Windows 8.1</th> <td>1.06%</td> <td>8.22%</td> <td style="color:green;">7.16</td> </tr> <tr class="odd"> <th scope="row">Macintosh OS X 10.5*</th> <td>–</td> <td>0.65%</td> <td style="color:green;">0.65</td> </tr> <tr class="even"> <th scope="row">Macintosh OS X 10.6*</th> <td>–</td> <td>2.90%</td> <td style="color:green;">2.90</td> </tr> <tr class="odd"> <th scope="row">Macintosh OS X 10.7*</th> <td>–</td> <td>1.91%</td> <td style="color:green;">1.91</td> </tr> <tr class="even"> <th scope="row">Macintosh OS X 10.8*</th> <td>–</td> <td>1.83%</td> <td style="color:green;">1.83</td> </tr> <tr class="odd"> <th scope="row">Macintosh OS X 10.9*</th> <td>–</td> <td>8.26%</td> <td style="color:green;">8.26</td> </tr> <tr class="even"> <th scope="row">Macintosh OS X 10.10</th> <td>0%</td> <td>5.69%</td> <td style="color:green;">5.69</td> </tr> <tr class="odd"> <th scope="row">iOS 4.3</th> <td>0.19%</td> <td>0%</td> <td style="color:red;">-0.19</td> </tr> <tr class="even"> <th scope="row">iOS 5.0</th> <td>0.12%</td> <td>0%</td> <td style="color:red;">-0.12</td> </tr> <tr class="odd"> <th scope="row">iOS 5.1</th> <td>0.59%</td> <td>0%</td> <td style="color:red;">-0.59</td> </tr> <tr class="even"> <th scope="row">iOS 6.0</th> <td>0.42%</td> <td>0%</td> <td style="color:red;">-0.42</td> </tr> <tr class="odd"> <th scope="row">iOS 6.1</th> <td>2.02%</td> <td>0.61%</td> <td style="color:red;">-1.41</td> </tr> <tr class="even"> <th scope="row">iOS 7.0</th> <td>7.36%</td> <td>1.54%</td> <td style="color:red;">-5.82</td> </tr> <tr class="odd"> <th scope="row">iOS 7.1</th> <td>0%</td> <td>5.76%</td> <td style="color:green;">5.76</td> </tr> <tr class="even"> <th scope="row">iOS 8.0</th> <td>0%</td> <td>3.27%</td> <td style="color:green;">3.27</td> </tr> <tr class="odd"> <th scope="row">Android 2.3</th> <td>0.77%</td> <td>0%</td> <td style="color:red;">-0.77</td> </tr> <tr class="even"> <th scope="row">Android 4.0</th> <td>0.82%</td> <td>0%</td> <td style="color:red;">-0.82</td> </tr> <tr class="odd"> <th scope="row">Android 4.1</th> <td>2.11%</td> <td>1.22%</td> <td style="color:red;">-0.89</td> </tr> <tr class="even"> <th scope="row">Android 4.2</th> <td>0.84%</td> <td>1.16%</td> <td style="color:green;">0.32</td> </tr> <tr class="odd"> <th scope="row">Android 4.3</th> <td>0.39%</td> <td>0.56%</td> <td style="color:green;">0.17</td> </tr> <tr class="even"> <th scope="row">Android 4.4</th> <td>0%</td> <td>3.80%</td> <td style="color:green;">3.80</td> </tr> <tr class="odd"> <th scope="row">Linux</th> <td>4.37%</td> <td>1.94%</td> <td style="color:red;">-2.43</td> </tr> </tbody> </table> <p>* We didn’t start breaking out individual versions of OS X until Q1 2014. So unfortunately for this post we don’t have great info breaking down the versions of OS X, but we will in the future. OS X 10.10 did not exist in Q1 2014 so it’s counted as a natural 0% in our Q1 data.</p> <h3>Browser version details</h3> <p>These are the most dynamic numbers of the bunch. If there’s one thing they prove, it’s how incredibly effective the upgrade policies of Chrome and Firefox are. Where Safari and IE have years-old versions still hanging on (I’m looking at you Safari 5.1 and IE 8.0), virtually every Chrome and Firefox user is using a browser released within the last six weeks. That’s a hugel powerful thing. The IE team has suggested that <a href="http://blogs.msdn.com/b/ie/archive/2015/01/22/project-spartan-and-the-windows-10-january-preview-build.aspx">Windows 10’s Project Spartan will adopt this policy</a>, which is absolutely fantastic news. A few highlights:</p> <ul> <li>Despite not being on a continuous upgrade cycle, Safari and IE were able to piggyback on successful OS launches to consolidate their users on their latest releases.</li> <li>IE 8.0 is the only non-latest version of IE still holding on, thanks to its status as the latest version available for the still somewhat popular Windows XP.</li> </ul> <table class="data"> <caption>OS versions in use on Flickr.com</caption> <thead> <tr> <th scope="col"></th> <th scope="col">2013 Q4</th> <th scope="col">2014 Q4</th> <th scope="col">Y/Y</th> </tr> </thead> <tbody> <tr class="odd"> <th scope="row">Chrome 22.0.1229</th> <td>1.67%</td> <td>0%</td> <td style="color:red;">-1.67</td> </tr> <tr class="even"> <th scope="row">Chrome 29.0.1547.76</th> <td>1.39%</td> <td>0%</td> <td style="color:red;">-1.39</td> </tr> <tr class="odd"> <th scope="row">Chrome 30.0.1599.101</th> <td>8.94%</td> <td>0%</td> <td style="color:red;">-8.94</td> </tr> <tr class="even"> <th scope="row">Chrome 30.0.1599.69</th> <td>3.74%</td> <td>0%</td> <td style="color:red;">-3.74</td> </tr> <tr class="odd"> <th scope="row">Chrome 31.0.1650.57</th> <td>6.08%</td> <td>0%</td> <td style="color:red;">-6.08</td> </tr> <tr class="even"> <th scope="row">Chrome 31.0.1650.63</th> <td>6.91%</td> <td>0%</td> <td style="color:red;">-6.91</td> </tr> <tr class="odd"> <th scope="row">Chrome 37.0.2062.124</th> <td>0%</td> <td>4.59%</td> <td style="color:green;">4.59</td> </tr> <tr class="even"> <th scope="row">Chrome 38.0.2125.104</th> <td>0%</td> <td>3.05%</td> <td style="color:green;">3.05</td> </tr> <tr class="odd"> <th scope="row">Chrome 38.0.2125.111</th> <td>0%</td> <td>6.65%</td> <td style="color:green;">6.65</td> </tr> <tr class="even"> <th scope="row">Chrome 39.0.2171.71</th> <td>0%</td> <td>4.09%</td> <td style="color:green;">4.09</td> </tr> <tr class="odd"> <th scope="row">Chrome 39.0.2171.95</th> <td>0%</td> <td>4.51%</td> <td style="color:green;">4.51</td> </tr> <tr class="even"> <th scope="row">Safari 5.0</th> <td>1.96%</td> <td>0%</td> <td style="color:red;">-1.96</td> </tr> <tr class="odd"> <th scope="row">Safari 5.1</th> <td>5.60%</td> <td>2.50%</td> <td style="color:red;">-3.10</td> </tr> <tr class="even"> <th scope="row">Safari 6.0</th> <td>6.21%</td> <td>0.86%</td> <td style="color:red;">-5.35</td> </tr> <tr class="odd"> <th scope="row">Safari 7.0</th> <td>7.29%</td> <td>7.25%</td> <td style="color:red;">-0.04</td> </tr> <tr class="even"> <th scope="row">Safari 7.1</th> <td>0%</td> <td>3.12%</td> <td style="color:green;">3.12</td> </tr> <tr class="odd"> <th scope="row">Safari 8.0</th> <td>0%</td> <td>10.10%</td> <td style="color:green;">10.10</td> </tr> <tr class="even"> <th scope="row">Firefox 22.0</th> <td>1.62%</td> <td>0%</td> <td style="color:red;">-1.62</td> </tr> <tr class="odd"> <th scope="row">Firefox 24.0</th> <td>5.50%</td> <td>0%</td> <td style="color:red;">-5.50</td> </tr> <tr class="even"> <th scope="row">Firefox 25.0</th> <td>6.46%</td> <td>0%</td> <td style="color:red;">-6.46</td> </tr> <tr class="odd"> <th scope="row">Firefox 26.0</th> <td>1.90%</td> <td>0%</td> <td style="color:red;">-1.90</td> </tr> <tr class="even"> <th scope="row">Firefox 32.0</th> <td>0%</td> <td>4.92%</td> <td style="color:green;">4.92</td> </tr> <tr class="odd"> <th scope="row">Firefox 33.0</th> <td>0%</td> <td>7.10%</td> <td style="color:green;">7.10</td> </tr> <tr class="even"> <th scope="row">Firefox 34.0</th> <td>0%</td> <td>3.52%</td> <td style="color:green;">3.52</td> </tr> <tr class="odd"> <th scope="row">MSIE 8.0</th> <td>3.69%</td> <td>1.00%</td> <td style="color:red;">-2.69</td> </tr> <tr class="even"> <th scope="row">MSIE 9.0</th> <td>3.04%</td> <td>1.22%</td> <td style="color:red;">-1.82</td> </tr> <tr class="odd"> <th scope="row">MSIE 10.0</th> <td>5.94%</td> <td>0%</td> <td style="color:red;">-5.94</td> </tr> <tr class="even"> <th scope="row">MSIE 11.0</th> <td>0%</td> <td>6.69%</td> <td style="color:green;">6.69</td> </tr> <tr class="odd"> <th scope="row">Generic WebKit 4.0*</th> <td>3.18%</td> <td>2.46%</td> <td style="color:red;">-0.72</td> </tr> <tr class="even"> <th scope="row">Mozilla 5.0*</th> <td>3.18%</td> <td>4.80%</td> <td style="color:green;">1.62</td> </tr> <tr class="odd"> <th scope="row">Opera 9.80</th> <td>1.46%</td> <td>0%</td> <td style="color:red;">-1.46</td> </tr> </tbody> </table> <p>* These are catch-all versions of Mozilla-based and Webkit-based browsers that aren’t themselves Firefox, Safari, or Chrome.</p> <h2>A word on methodology</h2> <p>These numbers were anonymously collected using Yahoo’s in-house metrics libraries. The numbers here are aggregated over the course of three months each, making these numbers lagging indicators. This is why the latest releases, like Android 5.0 and iOS 8.1, are under-represented – they hadn’t yet enjoyed one full quarter when 2014 came to a close.</p> <h2>Further reading</h2> <p>There are a number of excellent sites out there watching similar browser statistics on a continuing basis. A few of them are:</p> <ul> <li><a href='http://arstechnica.com/information-technology/2014/12/windows-7-up-as-windows-xp-slides-chrome-growth-stops-in-november/'>Ars Technica</a> – on a monthly basis they analyze raw data from Net Market Share with insightful commentary.</li> <li><a href='http://netmarketshare.com'>Net Market Share</a> – while Ars does a bang-up job, it’s helpful to sift the data yourself to find the answers to your questions.</li> <li>Peter-Paul Koch – No one shines a sharper light on the state of browsers than PPK, with just one example being his attention to disambiguating <a href='http://www.quirksmode.org/blog/archives/2015/02/chrome_continue.html'>the various versions of Chromium out there</a> (<a href='http://www.quirksmode.org/blog/archives/2015/02/counting_chromi.html'>part two</a>).</li> </ul> <p> <br /> </p> <div class="hiring-banner"> <p class="group-photo"> <a href="https://www.flickr.com/photos/captin_nod/14965686478/" title="Flickr September 2014 by Bhautik Joshi, on Flickr"><img decoding="async" loading="lazy" src="https://farm4.staticflickr.com/3893/14965686478_9278dbe39c_m.jpg" width="120" height="80" alt="Flickr September 2014"></a> </p> <p> Like this post? Have a love of online photography? Want to work with us? Flickr is hiring <strong>engineers</strong>, <strong>designers</strong> and <strong>product managers</strong> in our San Francisco office. <strong>Find out more at <a href="https://www.flickr.com/jobs/">flickr.com/jobs</a></strong>.</p> </div> </div> <footer class="entry-meta"> <span class="cat-links"> <span class="entry-utility-prep entry-utility-prep-cat-links">Posted in</span> <a href="https://code.flickr.net/category/metrics/" rel="category tag">metrics</a> </span> </footer> </article> <nav id="nav-below"> <h3 class="assistive-text">Post navigation</h3> <div class="nav-previous"><a href="https://code.flickr.net/page/3/" ><span class="meta-nav">←</span> Older posts</a></div> <div class="nav-next"><a href="https://code.flickr.net/" >Newer posts <span class="meta-nav">→</span></a></div> </nav> </div> </div> <div id="secondary" class="widget-area" role="complementary"> <aside id="jetpack-search-filters-3" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-3-wrapper" class="jetpack-instant-search-wrapper"> <div class="jetpack-search-form"> <form method="get" id="searchform" action="https://code.flickr.net/"> <label for="s" class="assistive-text">Search</label> <input type="text" class="field" name="s" id="s" placeholder="Search" /> <input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" /> <input type="hidden" name="orderby" value="" /><input type="hidden" name="order" value="" /></form> </div> <h4 class="jetpack-search-filters-widget__sub-heading"> Categories </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uncategorized"> Uncategorized (136) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="geo"> geo (12) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="kittens"> kittens (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="change-log"> changelog (7) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="category" data-val="uploadr"> uploadr (6) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Tags </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="api"> api (24) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="geo"> geo (13) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="machine-tags"> machine tags (10) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="javascript"> javascript (9) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="taxonomy" data-taxonomy="post_tag" data-val="kittentuesday"> kittentuesday (8) </a> </li> </ul> <h4 class="jetpack-search-filters-widget__sub-heading"> Year </h4> <ul class="jetpack-search-filters-widget__filter-list"> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2022-01-01 00:00:00" > 2022 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2021-01-01 00:00:00" > 2021 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2018-01-01 00:00:00" > 2018 (1) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2017-01-01 00:00:00" > 2017 (2) </a> </li> <li> <a href="#" class="jetpack-search-filter__link" data-filter-type="year_post_date" data-val="2016-01-01 00:00:00" > 2016 (5) </a> </li> </ul> </div></aside> <aside id="recent-posts-2" class="widget widget_recent_entries"> <h3 class="widget-title">Recent Posts</h3> <ul> <li> <a href="https://code.flickr.net/2022/02/14/safer-internet-day-and-open-source-codes-of-conduct/">Safer Internet Day and Open Source Codes of Conduct</a> </li> <li> <a href="https://code.flickr.net/2022/01/04/a-pluggable-solution-for-api-observability-on-our-php-system/">A Pluggable Solution for API Observability on our PHP System</a> </li> <li> <a href="https://code.flickr.net/2021/11/22/flickr-engineering-team-vision-guiding-principles/">Flickr Engineering Team Vision & Guiding Principles</a> </li> <li> <a href="https://code.flickr.net/2018/04/20/together/">Together</a> </li> <li> <a href="https://code.flickr.net/2017/03/07/introducing-similarity-search-at-flickr/">Introducing Similarity Search at Flickr</a> </li> </ul> </aside><aside id="archives-2" class="widget widget_archive"><h3 class="widget-title">Archives</h3> <ul> <li><a href='https://code.flickr.net/2022/02/'>February 2022</a></li> <li><a href='https://code.flickr.net/2022/01/'>January 2022</a></li> <li><a href='https://code.flickr.net/2021/11/'>November 2021</a></li> <li><a href='https://code.flickr.net/2018/04/'>April 2018</a></li> <li><a href='https://code.flickr.net/2017/03/'>March 2017</a></li> <li><a href='https://code.flickr.net/2017/01/'>January 2017</a></li> <li><a href='https://code.flickr.net/2016/09/'>September 2016</a></li> <li><a href='https://code.flickr.net/2016/05/'>May 2016</a></li> <li><a href='https://code.flickr.net/2016/04/'>April 2016</a></li> <li><a href='https://code.flickr.net/2016/03/'>March 2016</a></li> <li><a href='https://code.flickr.net/2015/12/'>December 2015</a></li> <li><a href='https://code.flickr.net/2015/11/'>November 2015</a></li> <li><a href='https://code.flickr.net/2015/09/'>September 2015</a></li> <li><a href='https://code.flickr.net/2015/07/'>July 2015</a></li> <li><a href='https://code.flickr.net/2015/06/'>June 2015</a></li> <li><a href='https://code.flickr.net/2015/03/'>March 2015</a></li> <li><a href='https://code.flickr.net/2014/10/'>October 2014</a></li> <li><a href='https://code.flickr.net/2014/08/'>August 2014</a></li> <li><a href='https://code.flickr.net/2014/07/'>July 2014</a></li> <li><a href='https://code.flickr.net/2014/05/'>May 2014</a></li> <li><a href='https://code.flickr.net/2014/04/'>April 2014</a></li> <li><a href='https://code.flickr.net/2014/02/'>February 2014</a></li> <li><a href='https://code.flickr.net/2013/09/'>September 2013</a></li> <li><a href='https://code.flickr.net/2013/06/'>June 2013</a></li> <li><a href='https://code.flickr.net/2013/03/'>March 2013</a></li> <li><a href='https://code.flickr.net/2012/12/'>December 2012</a></li> <li><a href='https://code.flickr.net/2012/10/'>October 2012</a></li> <li><a href='https://code.flickr.net/2012/07/'>July 2012</a></li> <li><a href='https://code.flickr.net/2012/06/'>June 2012</a></li> <li><a href='https://code.flickr.net/2012/05/'>May 2012</a></li> <li><a href='https://code.flickr.net/2012/04/'>April 2012</a></li> <li><a href='https://code.flickr.net/2012/02/'>February 2012</a></li> <li><a href='https://code.flickr.net/2012/01/'>January 2012</a></li> <li><a href='https://code.flickr.net/2011/12/'>December 2011</a></li> <li><a href='https://code.flickr.net/2011/10/'>October 2011</a></li> <li><a href='https://code.flickr.net/2011/09/'>September 2011</a></li> <li><a href='https://code.flickr.net/2011/08/'>August 2011</a></li> <li><a href='https://code.flickr.net/2011/07/'>July 2011</a></li> <li><a href='https://code.flickr.net/2011/06/'>June 2011</a></li> <li><a href='https://code.flickr.net/2011/03/'>March 2011</a></li> <li><a href='https://code.flickr.net/2011/02/'>February 2011</a></li> <li><a href='https://code.flickr.net/2011/01/'>January 2011</a></li> <li><a href='https://code.flickr.net/2010/11/'>November 2010</a></li> <li><a href='https://code.flickr.net/2010/10/'>October 2010</a></li> <li><a href='https://code.flickr.net/2010/09/'>September 2010</a></li> <li><a href='https://code.flickr.net/2010/08/'>August 2010</a></li> <li><a href='https://code.flickr.net/2010/07/'>July 2010</a></li> <li><a href='https://code.flickr.net/2010/05/'>May 2010</a></li> <li><a href='https://code.flickr.net/2010/04/'>April 2010</a></li> <li><a href='https://code.flickr.net/2010/03/'>March 2010</a></li> <li><a href='https://code.flickr.net/2010/02/'>February 2010</a></li> <li><a href='https://code.flickr.net/2010/01/'>January 2010</a></li> <li><a href='https://code.flickr.net/2009/12/'>December 2009</a></li> <li><a href='https://code.flickr.net/2009/11/'>November 2009</a></li> <li><a href='https://code.flickr.net/2009/10/'>October 2009</a></li> <li><a href='https://code.flickr.net/2009/09/'>September 2009</a></li> <li><a href='https://code.flickr.net/2009/07/'>July 2009</a></li> <li><a href='https://code.flickr.net/2009/06/'>June 2009</a></li> <li><a href='https://code.flickr.net/2009/05/'>May 2009</a></li> <li><a href='https://code.flickr.net/2009/04/'>April 2009</a></li> <li><a href='https://code.flickr.net/2009/03/'>March 2009</a></li> <li><a href='https://code.flickr.net/2009/02/'>February 2009</a></li> <li><a href='https://code.flickr.net/2009/01/'>January 2009</a></li> <li><a href='https://code.flickr.net/2008/12/'>December 2008</a></li> <li><a href='https://code.flickr.net/2008/11/'>November 2008</a></li> <li><a href='https://code.flickr.net/2008/10/'>October 2008</a></li> <li><a href='https://code.flickr.net/2008/09/'>September 2008</a></li> <li><a href='https://code.flickr.net/2008/08/'>August 2008</a></li> <li><a href='https://code.flickr.net/2008/07/'>July 2008</a></li> <li><a href='https://code.flickr.net/2008/06/'>June 2008</a></li> <li><a href='https://code.flickr.net/2008/05/'>May 2008</a></li> <li><a href='https://code.flickr.net/2008/04/'>April 2008</a></li> </ul> </aside><aside id="categories-2" class="widget widget_categories"><h3 class="widget-title">Categories</h3> <ul> <li class="cat-item cat-item-11749740"><a href="https://code.flickr.net/category/api-2/">API</a> </li> <li class="cat-item cat-item-564792"><a href="https://code.flickr.net/category/change-log/">changelog</a> </li> <li class="cat-item cat-item-5784"><a href="https://code.flickr.net/category/event/">event</a> </li> <li class="cat-item cat-item-29160"><a href="https://code.flickr.net/category/geo/">geo</a> </li> <li class="cat-item cat-item-139037766"><a href="https://code.flickr.net/category/hadoop/">hadoop</a> </li> <li class="cat-item cat-item-32"><a href="https://code.flickr.net/category/infrastructure/">infrastructure</a> </li> <li class="cat-item cat-item-139037765"><a href="https://code.flickr.net/category/kittens/">kittens</a> </li> <li class="cat-item cat-item-20156"><a href="https://code.flickr.net/category/labs/">labs</a> </li> <li class="cat-item cat-item-171"><a href="https://code.flickr.net/category/meta/">meta</a> </li> <li class="cat-item cat-item-7092"><a href="https://code.flickr.net/category/metrics/">metrics</a> </li> <li class="cat-item cat-item-139037764"><a href="https://code.flickr.net/category/open-source/">open source</a> </li> <li class="cat-item cat-item-1930"><a href="https://code.flickr.net/category/performance/">performance</a> </li> <li class="cat-item cat-item-304"><a href="https://code.flickr.net/category/photos/">photos</a> </li> <li class="cat-item cat-item-2373"><a href="https://code.flickr.net/category/search/">search</a> </li> <li class="cat-item cat-item-1"><a href="https://code.flickr.net/category/uncategorized/">Uncategorized</a> </li> <li class="cat-item cat-item-249276"><a href="https://code.flickr.net/category/uploadr/">uploadr</a> </li> <li class="cat-item cat-item-412"><a href="https://code.flickr.net/category/video/">video</a> </li> <li class="cat-item cat-item-830560"><a href="https://code.flickr.net/category/xulrunner/">xulrunner</a> </li> </ul> </aside><aside id="meta-2" class="widget widget_meta"><h3 class="widget-title">Meta</h3> <ul> <li><a href="https://code.flickr.net/wp-login.php">Log in</a></li> <li><a href="https://code.flickr.net/feed/">Entries feed</a></li> <li><a href="https://code.flickr.net/comments/feed/">Comments feed</a></li> <li><a href="https://wordpress.org/">WordPress.org</a></li> </ul> </aside> </div> </div> <footer id="colophon" role="contentinfo"> <div id="site-generator"> © 2024 Flickr, Inc. All rights reserved. | Powered by <a href="https://wpvip.com/?utm_source=vip_powered_wpcom&utm_medium=web&utm_campaign=VIP%20Footer%20Credit&utm_term=code.flickr.net" rel="generator nofollow" class="powered-by-wpcom">WordPress VIP</a> </div> </footer> </div> <div class="jetpack-instant-search__widget-area" style="display: none"> <div id="jetpack-search-filters-2" class="widget jetpack-filters widget_search"> <div id="jetpack-search-filters-2-wrapper" class="jetpack-instant-search-wrapper"> </div></div> </div> <script type="text/javascript" src="https://code.flickr.net/_static/??-eJzTLy/QTc7PK0nNK9EvyClNz8wr1i+uzCtJrMjITM/IAeKS1CJMEWP94uSizIISoOIM5/yiVL2sYh19oowqycgsStEtSCwqqdRNKiotzkgFmeEEYvknZTkTbxBeN4HNC8hJzMwDGmifa2tobmRoaGhiYGqYBQABb1yv" ></script><script type='text/javascript'> (function(){ var corecss = document.createElement('link'); var themecss = document.createElement('link'); var corecssurl = "https://code.flickr.net/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shCore.css?ver=3.0.9b"; if ( corecss.setAttribute ) { corecss.setAttribute( "rel", "stylesheet" ); corecss.setAttribute( "type", "text/css" ); corecss.setAttribute( "href", corecssurl ); } else { corecss.rel = "stylesheet"; corecss.href = corecssurl; } document.head.appendChild( corecss ); var themecssurl = "https://code.flickr.net/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shThemeDefault.css?ver=3.0.9b"; if ( themecss.setAttribute ) { themecss.setAttribute( "rel", "stylesheet" ); themecss.setAttribute( "type", "text/css" ); themecss.setAttribute( "href", themecssurl ); } else { themecss.rel = "stylesheet"; themecss.href = themecssurl; } document.head.appendChild( themecss ); })(); SyntaxHighlighter.config.strings.expandSource = '+ expand source'; SyntaxHighlighter.config.strings.help = '?'; SyntaxHighlighter.config.strings.alert = 'SyntaxHighlighter\n\n'; SyntaxHighlighter.config.strings.noBrush = 'Can\'t find brush for: '; SyntaxHighlighter.config.strings.brushNotHtmlScript = 'Brush wasn\'t configured for html-script option: '; SyntaxHighlighter.defaults['pad-line-numbers'] = false; SyntaxHighlighter.defaults['toolbar'] = false; SyntaxHighlighter.all(); // Infinite scroll support if ( typeof( jQuery ) !== 'undefined' ) { jQuery( function( $ ) { $( document.body ).on( 'post-load', function() { SyntaxHighlighter.highlight(); } ); } ); } </script> <script type="text/javascript" src="https://code.flickr.net/wp-includes/js/dist/url.min.js?m=1732205990g" ></script><script id="jetpack-instant-search-js-before" type="text/javascript"> var JetpackInstantSearchOptions=JSON.parse(decodeURIComponent("%7B%22overlayOptions%22%3A%7B%22colorTheme%22%3A%22light%22%2C%22enableInfScroll%22%3Atrue%2C%22enableFilteringOpensOverlay%22%3Atrue%2C%22enablePostDate%22%3Atrue%2C%22enableSort%22%3Atrue%2C%22highlightColor%22%3A%22%23FFC%22%2C%22overlayTrigger%22%3A%22submit%22%2C%22resultFormat%22%3A%22expanded%22%2C%22showPoweredBy%22%3Atrue%2C%22defaultSort%22%3A%22relevance%22%2C%22excludedPostTypes%22%3A%5B%5D%7D%2C%22homeUrl%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%22%2C%22locale%22%3A%22en-US%22%2C%22postsPerPage%22%3A10%2C%22siteId%22%3A185426273%2C%22postTypes%22%3A%7B%22post%22%3A%7B%22singular_name%22%3A%22Post%22%2C%22name%22%3A%22Posts%22%7D%2C%22page%22%3A%7B%22singular_name%22%3A%22Page%22%2C%22name%22%3A%22Pages%22%7D%2C%22attachment%22%3A%7B%22singular_name%22%3A%22Media%22%2C%22name%22%3A%22Media%22%7D%7D%2C%22webpackPublicPath%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-content%5C%2Fmu-plugins%5C%2Fjetpack-13.1%5C%2Fjetpack_vendor%5C%2Fautomattic%5C%2Fjetpack-search%5C%2Fbuild%5C%2Finstant-search%5C%2F%22%2C%22isPhotonEnabled%22%3Afalse%2C%22isFreePlan%22%3Afalse%2C%22apiRoot%22%3A%22https%3A%5C%2F%5C%2Fcode.flickr.net%5C%2Fwp-json%5C%2F%22%2C%22apiNonce%22%3A%22fbe66387bb%22%2C%22isPrivateSite%22%3Afalse%2C%22isWpcom%22%3Afalse%2C%22hasOverlayWidgets%22%3Atrue%2C%22widgets%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_0%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_1%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22taxonomy_2%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%2C%22filter_id%22%3A%22date_histogram_3%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-2%22%7D%5D%2C%22widgetsOutsideOverlay%22%3A%5B%7B%22filters%22%3A%5B%7B%22name%22%3A%22Bylines%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22byline%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_4%22%7D%2C%7B%22name%22%3A%22Categories%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22category%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_5%22%7D%2C%7B%22name%22%3A%22Tags%22%2C%22type%22%3A%22taxonomy%22%2C%22taxonomy%22%3A%22post_tag%22%2C%22count%22%3A5%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22taxonomy_6%22%7D%2C%7B%22name%22%3A%22Year%22%2C%22type%22%3A%22date_histogram%22%2C%22count%22%3A5%2C%22field%22%3A%22post_date%22%2C%22interval%22%3A%22year%22%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%2C%22filter_id%22%3A%22date_histogram_7%22%7D%5D%2C%22widget_id%22%3A%22jetpack-search-filters-3%22%7D%5D%2C%22hasNonSearchWidgets%22%3Afalse%2C%22preventTrackingCookiesReset%22%3Afalse%7D")); </script> <script src='https://code.flickr.net/wp-content/mu-plugins/jetpack-13.1/jetpack_vendor/automattic/jetpack-search/build/instant-search/jp-search.js?minify=false&ver=32fdf369306ecec73d70' id='jetpack-instant-search-js'></script> <script src='//stats.wp.com/w.js?ver=202447' id='jp-tracks-js'></script> <script src='https://stats.wp.com/e-202447.js' id='jetpack-stats-js' data-wp-strategy='defer'></script> <script id="jetpack-stats-js-after" type="text/javascript"> _stq = window._stq || []; _stq.push([ "view", JSON.parse("{\"v\":\"ext\",\"blog\":\"185426273\",\"post\":\"0\",\"tz\":\"-8\",\"srv\":\"code.flickr.net\",\"hp\":\"vip\",\"j\":\"1:13.1.4\"}") ]); _stq.push([ "clickTrackerInit", "185426273", "0" ]); </script> <script async src="https://embedr.flickr.com/assets/client-code.js" charset="utf-8"></script> </body> </html>