CINXE.COM

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">     <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en" xmlns:og="http://ogp.me/ns#" xmlns:fb="http://ogp.me/ns/fb#"> <head> <title>Ceph-based storage services for Run2 and beyond - CERN Document Server</title> <link href='https://framework.web.cern.ch/framework/2.0/fonts/PTSansWeb/PTSansWeb.css' rel='stylesheet' type='text/css' /> <link rel="stylesheet" href="https://cds.cern.ch/img/invenio.css?v=20141127" type="text/css" /> <link rel="stylesheet" href="https://cds.cern.ch/img/cern_theme/css/cern_theme.css?v=20141127" type="text/css" /> <link rel="stylesheet"href="/css/font-awesome.min.css"> <meta http-equiv="X-UA-Compatible" content="IE=Edge"/> <link rel="stylesheet" href="https://cds.cern.ch/img/cern_toolbar/css/toolbar.css" type="text/css" />    <link rel="canonical" href="https://cds.cern.ch/record/2134576" /> <link rel="alternate" hreflang="el" href="https://cds.cern.ch/record/2134576?ln=el" /> <link rel="alternate" hreflang="fr" href="https://cds.cern.ch/record/2134576?ln=fr" /> <link rel="alternate" hreflang="bg" href="https://cds.cern.ch/record/2134576?ln=bg" /> <link rel="alternate" hreflang="zh-TW" href="https://cds.cern.ch/record/2134576?ln=zh_TW" /> <link rel="alternate" hreflang="pt" href="https://cds.cern.ch/record/2134576?ln=pt" /> <link rel="alternate" hreflang="no" href="https://cds.cern.ch/record/2134576?ln=no" /> <link rel="alternate" hreflang="hr" href="https://cds.cern.ch/record/2134576?ln=hr" /> <link rel="alternate" hreflang="ca" href="https://cds.cern.ch/record/2134576?ln=ca" /> <link rel="alternate" hreflang="de" href="https://cds.cern.ch/record/2134576?ln=de" /> <link rel="alternate" hreflang="it" href="https://cds.cern.ch/record/2134576?ln=it" /> <link rel="alternate" hreflang="zh-CN" href="https://cds.cern.ch/record/2134576?ln=zh_CN" /> <link rel="alternate" hreflang="sv" href="https://cds.cern.ch/record/2134576?ln=sv" /> <link rel="alternate" hreflang="sk" href="https://cds.cern.ch/record/2134576?ln=sk" /> <link rel="alternate" hreflang="en" href="https://cds.cern.ch/record/2134576?ln=en" /> <link rel="alternate" hreflang="pl" href="https://cds.cern.ch/record/2134576?ln=pl" /> <link rel="alternate" hreflang="ru" href="https://cds.cern.ch/record/2134576?ln=ru" /> <link rel="alternate" hreflang="ka" href="https://cds.cern.ch/record/2134576?ln=ka" /> <link rel="alternate" hreflang="ja" href="https://cds.cern.ch/record/2134576?ln=ja" /> <link rel="alternate" hreflang="es" href="https://cds.cern.ch/record/2134576?ln=es" /> <link rel="alternate" type="application/rss+xml" title="CERN Document Server RSS" href="/rss?ln=en" /> <link rel="search" type="application/opensearchdescription+xml" href="https://cds.cern.ch/opensearchdescription" title="CERN Document Server" /> <link rel="unapi-server" type="application/xml" title="unAPI" href="https://cds.cern.ch/unapi" /> <link rel="apple-touch-icon" href="/apple-touch-icon.png"/> <link rel="apple-touch-icon-precomposed" href="/apple-touch-icon-precomposed.png"/> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta http-equiv="Content-Language" content="en" /> <meta name="description" content="In 2013, CERN IT evaluated then deployed a petabyte-scale Ceph cluster to support OpenStack use-cases in production. With now more than a year of smooth operations, we will present our experience and tuning best-practices. Beyond the cloud storage use-cases, we have been exploring Ceph-based services to satisfy the growing storage requirements during and after Run2. First, we have developed a Ceph back-end for CASTOR, allowing this service to deploy thin disk server nodes which act as gateways to Ceph, this feature marries the strong data archival and cataloging features of CASTOR with the resilient and high performance Ceph subsystem for disk. Second, we have developed RADOSFS, a lightweight storage API which builds a POSIX-like filesystem on top of the Ceph object layer. When combined with Xrootd, RADOSFS can offer a scalable object interface compatible with our HEP data processing applications. Lastly the same object layer is being used to build a scalable and inexpensive NFS service for several user communities. van der Ster, Daniel C; Lamanna, Massimo; Mascetti, Luca; Peters, Andreas J; Rousseau, Herv" /> <meta name="keywords" content="CERN Document Server, WebSearch, CERN Document Server" /> <script type="text/javascript" src="https://cds.cern.ch/js/jquery.min.js"></script>  <link rel="stylesheet" href="https://cds.cern.ch/img/webnews.css" type="text/css" />  <script type="text/javascript" src="https://cds.cern.ch/js/webnews.js?v=20131009"></script> <meta property="fb:app_id" content="137353533001720"/> <script type="text/x-mathjax-config"> MathJax.Hub.Config({ tex2jax: {inlineMath: [['$','$']], processEscapes: true}, showProcessingMessages: false, messageStyle: "none" }); </script> <script src="/MathJax/MathJax.js?config=TeX-AMS_CHTML" type="text/javascript"> </script>  <meta content="Ceph-based storage services for Run2 and beyond" name="citation_title" /> <meta content="van der Ster, Daniel C" name="citation_author" /> <meta content="Peters, Andreas J" name="citation_author" /> <meta content="Mascetti, Luca" name="citation_author" /> <meta content="Rousseau, Herv" name="citation_author" /> <meta content="Lamanna, Massimo" name="citation_author" /> <meta content="10.1088/1742-6596/664/4/042054" name="citation_doi" /> <meta content="J. Phys.: Conf. Ser." name="citation_journal_title" /> <meta content="664" name="citation_volume" /> <meta content="042054" name="citation_firstpage" /> <meta content="2015" name="citation_publication_date" /> <meta name="citation_online_date" content="2016/02/26"> <meta content="10.1088/1742-6596/664/4/042054" name="citation_doi" /> <meta name="citation_pdf_url" content="https://cds.cern.ch/record/2134576/files/Ceph-based storage services for Run2 and beyond.pdf" />  <meta content="Ceph-based storage services for Run2 and beyond" property="og:title" /> <meta content="website" property="og:type" /> <meta content="website" property="og:type" /> <meta content="https://cds.cern.ch/record/2134576" property="og:url" /> <meta content="CERN Document Server" property="og:site_name" /> <meta content="In 2013, CERN IT evaluated then deployed a petabyte-scale Ceph cluster to support OpenStack use-cases in production. With now more than a year of smooth operations, we will present our experience and tuning best-practices. Beyond the cloud storage use-cases, we have been exploring Ceph-based services to satisfy the growing storage requirements during and after Run2. First, we have developed a Ceph back-end for CASTOR, allowing this service to deploy thin disk server nodes which act as gateways to Ceph, this feature marries the strong data archival and cataloging features of CASTOR with the resilient and high performance Ceph subsystem for disk. Second, we have developed RADOSFS, a lightweight storage API which builds a POSIX-like filesystem on top of the Ceph object layer. When combined with Xrootd, RADOSFS can offer a scalable object interface compatible with our HEP data processing applications. Lastly the same object layer is being used to build a scalable and inexpensive NFS service for several user communities." property="og:description" /> <meta content="IOP" property="og:description" />  <meta content="summary" name="twitter:card" /> <meta content="summary" name="twitter:card" /> <style></style> </head> <body class="CERN32Document32Server search" lang="en">  <div id="cern-toolbar"> <h1><a href="http://cern.ch" title="CERN">CERN <span>Accelerating science</span></a></h1> <ul> <li class="cern-accountlinks"><a class="cern-account" href="https://cds.cern.ch/youraccount/login?ln=en&referer=https%3A//cds.cern.ch/record/2134576%3Fln%3Den" title="Sign in to your CERN account">Sign in</a></li> <li><a class="cern-directory" href="http://cern.ch/directory" title="Search CERN resources and browse the directory">Directory</a></li> </ul> </div>   <div role="banner" class="clearfix" id="header"> <div class="header-inner inner"> <hgroup class="clearfix"> <h2 id="site-name"> <a rel="home" title="Home" href="/"><span>CERN Document Server</span></a> </h2> <h3 id="site-slogan">Access articles, reports and multimedia content in HEP</h3> </hgroup> <div role="navigation" id="main-navigation" class="cdsmenu"> <h2 class="element-invisible">Main menu</h2><ul class="links inline clearfix"> <li class="menu-386 first active-trail"><a class="active-trail" href="https://cds.cern.ch/?ln=en">Search</a></li> <li class="menu-444 "><a class="" title="" href="https://cds.cern.ch/submit?ln=en">Submit</a></li> <li class="menu-426 "><a class="" href="https://cds.cern.ch/help/?ln=en">Help</a></li> <li class="leaf hassubcdsmenu"> <a hreflang="en" class="header" href="https://cds.cern.ch/youraccount/display?ln=en">Personalize</a> <ul class="subsubcdsmenu"><li><a href="https://cds.cern.ch/youralerts/list?ln=en">Your alerts</a></li><li><a href="https://cds.cern.ch/yourbaskets/display?ln=en">Your baskets</a></li><li><a href="https://cds.cern.ch/yourcomments?ln=en">Your comments</a></li><li><a href="https://cds.cern.ch/youralerts/display?ln=en">Your searches</a></li></ul></li> </ul> </div> </div> </div>  <table class="navtrailbox"> <tr> <td class="navtrailboxbody"> <a href="/?ln=en" class="navtrail">Home</a> > Ceph-based storage services for Run2 and beyond </td> </tr> </table> </div> <div class="pagebody"><div class="pagebodystripemiddle"> <div class="detailedrecordbox"> <div class="detailedrecordtabs"> <div> <ul class="detailedrecordtabs"><li class="on first"><a href="/record/2134576/?ln=en">Information </a></li><li class=""><a href="/record/2134576/files?ln=en">Files </a></li></ul> <div id="tabsSpacer" style="clear:both;height:0px"> </div></div> </div> <div class="detailedrecordboxcontent"> <div class="top-left-folded"></div> <div class="top-right-folded"></div> <div class="inside">  <abbr class="unapi-id" title="2134576"></abbr> <style type="text/css">  </style> <table class="formatRecordTableFullWidth" > <tr> <td class="formatRecordHeader" style="background-image: url('https://cds.cern.ch/img/journals.jpg');" colspan="2">  <a style="color:#fff;text-decoration:none" href="https://cds.cern.ch/collection/Published%20Articles">Published Articles</a> </td> </tr> <script type="text/javascript"> $( document ).ready(function() { $('.showAuthor').on('click', function() { var author = '<p>' + $(this).data('name') + '</p>'; var affiliation = $(this).data('affiliation') + '</br>'; var contribution = $(this).data('contribution') + '</br>'; $.magnificPopup.open({ items: { src: '<div id="ovelary-mathjax" class="overlay-white oc-content overlay-white-500">' + author + affiliation + contribution + '</div>', type: 'inline' }, callbacks: { open: function() { var div = document.getElementById("overlay-mathjax") MathJax.Hub.Queue(["Typeset", MathJax.Hub, div]); }, } }) }) }); </script> <tr><td class="formatRecordLabel"> Title </td><td style="padding-left:5px;"><b>Ceph-based storage services for Run2 and beyond</b></td></tr> <tr><td class="formatRecordLabel"><span style="white-space:nowrap;"> Author(s) </span> </td><td style="padding-left:5px;"><a href="https://cds.cern.ch/search?f=author&p=van%20der%20Ster%2C%20Daniel%20C&ln=en">van der Ster, Daniel C</a> (CERN) ; <a href="https://cds.cern.ch/search?f=author&p=Lamanna%2C%20Massimo&ln=en">Lamanna, Massimo</a> (CERN) ; <a href="https://cds.cern.ch/search?f=author&p=Mascetti%2C%20Luca&ln=en">Mascetti, Luca</a> (CERN) ; <a href="https://cds.cern.ch/search?f=author&p=Peters%2C%20Andreas%20J&ln=en">Peters, Andreas J</a> (CERN) ; <a href="https://cds.cern.ch/search?f=author&p=Rousseau%2C%20Herv&ln=en">Rousseau, Herv</a> (CERN)</td></tr> <tr><td class="formatRecordLabel"> Publication </td><td style="padding-left:5px;">2015</td></tr> <tr><td class="formatRecordLabel"> Number of pages </td><td style="padding-left:5px;">4</td></tr> <tr><td class="formatRecordLabel"> In: </td><td style="padding-left:5px;"><a href="http://dx.doi.org/10.1088/1742-6596/664/4/042054"><i>J. Phys.: Conf. Ser.</i> 664 (2015) 042054</a> </a></td></tr> <tr><td class="formatRecordLabel"> In: </td><td style="padding-left:5px;"><a href="https://cds.cern.ch/record/1958286">21st International Conference on Computing in High Energy and Nuclear Physics</a>, Okinawa, Japan, 13 - 17 Apr 2015, pp.042054</td></tr> <tr><td class="formatRecordLabel"> DOI </td><td style="padding-left:5px;"><a href="http://dx.doi.org/10.1088/1742-6596/664/4/042054" title="DOI" target="_blank">10.1088/1742-6596/664/4/042054</a> <tr><td class="formatRecordLabel"> Subject category </td><td style="padding-left:5px;">Computing and Computers</td></tr> <tr><td class="formatRecordLabel"> Abstract </td><td style="padding-left:5px;">In 2013, CERN IT evaluated then deployed a petabyte-scale Ceph cluster to support OpenStack use-cases in production. With now more than a year of smooth operations, we will present our experience and tuning best-practices. Beyond the cloud storage use-cases, we have been exploring Ceph-based services to satisfy the growing storage requirements during and after Run2. First, we have developed a Ceph back-end for CASTOR, allowing this service to deploy thin disk server nodes which act as gateways to Ceph, this feature marries the strong data archival and cataloging features of CASTOR with the resilient and high performance Ceph subsystem for disk. Second, we have developed RADOSFS, a lightweight storage API which builds a POSIX-like filesystem on top of the Ceph object layer. When combined with Xrootd, RADOSFS can offer a scalable object interface compatible with our HEP data processing applications. Lastly the same object layer is being used to build a scalable and inexpensive NFS service for several user communities.</td></tr> <tr><td class="formatRecordLabel"> Copyright/License </td><td style="padding-left:5px;">publication: © 2015-2025 The Author(s) (License: <a href="http://creativecommons.org/licenses/by/3.0/">CC-BY-3.0</a>)</td></tr> </table> <br/>Corresponding record in: <a href="http://inspirehep.net/record/1413881">Inspire</a> <small> </small> <br/> <br/><br/><div align="right"><div style="padding-bottom:2px;padding-top:30px;"><span class="moreinfo" style="margin-right:10px;"> <a href="" class="moreinfo">Back to search</a> </span></div></div> <div class="bottom-left-folded"><div class="recordlastmodifiedbox" style="position:relative;margin-left:1px"> Record created 2016-02-26, last modified 2022-08-10</div></div> <div class="bottom-right-folded" style="text-align:right;padding-bottom:2px;"> <span class="moreinfo" style="margin-right:10px;"><a href="/search?ln=en&p=recid%3A2134576&rm=wrd" class="moreinfo">Similar records</a></span></div> </div> </div> </div> <br/> <br /> <div class="detailedrecordminipanel"> <div class="top-left"></div><div class="top-right"></div> <div class="inside"> <div id="detailedrecordminipanelfile" style="width:33%;float:left;text-align:center;margin-top:0"> <div><small class="detailedRecordActions">IOP Open Access article:</small> <br /><a href="/record/2134576/files/Ceph-based storage services for Run2 and beyond.pdf"><img style="border:none" src="/img/file-icon-text-34x48.gif" alt="Download fulltext" /><br />PDF</a><br /></div> </div> <div id="detailedrecordminipanelreview" style="width:30%;float:left;text-align:center"> </div> <div id="detailedrecordminipanelactions" style="width:36%;float:right;text-align:right;"> <ul class="detailedrecordactions"> <li><a href="/yourbaskets/add?ln=en&recid=2134576">Add to personal basket</a></li> <li>Export as <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/hx?ln=en">BibTeX</a>, <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/hm?ln=en">MARC</a>, <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/xm?ln=en">MARCXML</a>, <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/xd?ln=en">DC</a>, <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/xe?ln=en">EndNote</a>,  <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/xn?ln=en">NLM</a>, <a style="text-decoration:underline;font-weight:normal" href="/record/2134576/export/xw?ln=en">RefWorks</a> </li> </ul> <div style='padding-left: 13px;'>  <div id="bookmark"></div> <div id="bookmark_sciencewise"></div> <style type="text/css"> #bookmark_sciencewise, #bookmark {float: left;} #bookmark_sciencewise li {padding: 2px; width: 25px;} #bookmark_sciencewise ul, #bookmark ul {list-style-image: none;} </style> <script type="text/javascript" src="/js/jquery.bookmark.min.js"></script> <style type="text/css">@import "/css/jquery.bookmark.css";</style> <script type="text/javascript">// <![CDATA[ $.bookmark.addSite('sciencewise', 'ScienceWise.info', 'https://cds.cern.ch/img/sciencewise.png', 'en', 'bookmark', 'http://sciencewise.info/bookmarks/cds:2134576/add'); $('#bookmark_sciencewise').bookmark({sites: ['sciencewise']}); $('#bookmark').bookmark({ sites: ['facebook', 'twitter', 'linkedin', 'google_plusone'], icons: '/img/bookmarks.png', url: 'https://cds.cern.ch/record/2134576', addEmail: true, title: "Ceph-based storage services for Run2 and beyond", description: "In 2013, CERN IT evaluated then deployed a petabyte-scale Ceph cluster to support OpenStack use-cases in production. With now more than a year of smooth operations, we will present our experience and tuning best-practices. Beyond the cloud storage use-cases, we have been exploring Ceph-based services to satisfy the growing storage requirements during and after Run2. First, we have developed a Ceph back-end for CASTOR, allowing this service to deploy thin disk server nodes which act as gateways to Ceph, this feature marries the strong data archival and cataloging features of CASTOR with the resilient and high performance Ceph subsystem for disk. Second, we have developed RADOSFS, a lightweight storage API which builds a POSIX-like filesystem on top of the Ceph object layer. When combined with Xrootd, RADOSFS can offer a scalable object interface compatible with our HEP data processing applications. Lastly the same object layer is being used to build a scalable and inexpensive NFS service for several user communities." }); // ]]> </script>  </div> </div> <div style="clear:both;margin-bottom: 0;"></div> </div> <div class="bottom-left"></div><div class="bottom-right"></div> </div> </div></div> <footer id="footer" class="pagefooter clearfix">  <div class="pagefooterstripeleft"> CERN Document Server :: <a class="footer" href="https://cds.cern.ch/?ln=en">Search</a> :: <a class="footer" href="https://cds.cern.ch/submit?ln=en">Submit</a> :: <a class="footer" href="https://cds.cern.ch/youraccount/display?ln=en">Personalize</a> :: <a class="footer" href="https://cds.cern.ch/help/?ln=en">Help</a> :: <a class="footer" href="https://cern.service-now.com/service-portal?id=privacy_policy&se=CDS-Service" target="_blank">Privacy Notice</a> :: <a class="footer" href="https://repository.cern/content-policy" target="_blank">Content Policy</a> :: <a class="footer" href="https://repository.cern/terms" target="_blank">Terms and Conditions</a> <br /> Powered by <a class="footer" href="http://invenio-software.org/">Invenio</a> <br /> Maintained by <a class="footer" href="https://cern.service-now.com/service-portal?id=service_element&name=CDS-Service">CDS Service</a> - Need help? Contact <a href="https://cern.service-now.com/service-portal?id=service_element&name=CDS-Service">CDS Support</a>. <br /> </div> <div class="pagefooterstriperight"> <div class="cern-logo"> <a id="logo" href="http://cern.ch" title="CERN" rel="CERN" ><img src="https://cds.cern.ch/img/cern_theme/img/cern-logo-large.png" alt="CERN" /></a> </div> <div class="cern-languagebox"> This site is also available in the following languages:<br /><a href="/record/2134576?ln=bg" class="langinfo">Български</a>  <a href="/record/2134576?ln=ca" class="langinfo">Català</a>  <a href="/record/2134576?ln=de" class="langinfo">Deutsch</a>  <a href="/record/2134576?ln=el" class="langinfo">Ελληνικά</a>  <span class="langinfo">English</span>  <a href="/record/2134576?ln=es" class="langinfo">Español</a>  <a href="/record/2134576?ln=fr" class="langinfo">Français</a>  <a href="/record/2134576?ln=hr" class="langinfo">Hrvatski</a>  <a href="/record/2134576?ln=it" class="langinfo">Italiano</a>  <a href="/record/2134576?ln=ja" class="langinfo">日本語</a>  <a href="/record/2134576?ln=ka" class="langinfo">ქართული</a>  <a href="/record/2134576?ln=no" class="langinfo">Norsk/Bokmål</a>  <a href="/record/2134576?ln=pl" class="langinfo">Polski</a>  <a href="/record/2134576?ln=pt" class="langinfo">Português</a>  <a href="/record/2134576?ln=ru" class="langinfo">Русский</a>  <a href="/record/2134576?ln=sk" class="langinfo">Slovensky</a>  <a href="/record/2134576?ln=sv" class="langinfo">Svenska</a>  <a href="/record/2134576?ln=zh_CN" class="langinfo">中文(简)</a>  <a href="/record/2134576?ln=zh_TW" class="langinfo">中文(繁)</a> </div> </div>  </footer> <script type="text/javascript"> var SyndeticsBookCovers = (function() { var SMALL_SIZE = "sc.gif", MEDIUM_SIZE = "mc.gif", RAW_URL = "https://secure.syndetics.com/index.aspx?isbn=THEISBN/THESIZE&client=cernlibrary"; replaceCover = function(imgElement, isbns, hdFormat) { var img = new Image(), size = hdFormat ? MEDIUM_SIZE : SMALL_SIZE; var _isbns = isbns.sort(function(a, b) { // sort from shortest to longest ISBN (more modern) return a.length > b.length ? 1 : -1; }); function next() { var isbn = _isbns.pop(); if (isbn) { var url = RAW_URL.replace("THEISBN", isbn).replace("THESIZE", size); img.src = url; } } function done() { imgElement.src = img.src; } img.onload = function() { if (this.width > 1) { done(); } else { next(); } }; next(); }; return { replaceCover: replaceCover }; })(); $(document).ready(function() { // get book covers $("img.book-cover").each(function() { var $this = $(this), strIsbns = $this.data("isbns") || "", isbnsArray = String(strIsbns).split(","), hdFormat = $this.hasClass("hd"); SyndeticsBookCovers.replaceCover(this, isbnsArray, hdFormat); }); // WebNews tooltips $.ajax({ url: "/news/tooltips", success: function(data) { create_tooltips(data); }, dataType: "json", cache: false }); }); </script>  <script src="//cds.cern.ch/js/feedback.js"></script>   <script> var _paq = window._paq = window._paq || []; /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ _paq.push(['trackPageView']); _paq.push(['enableLinkTracking']); (function() { var u="https://webanalytics.web.cern.ch/"; _paq.push(['setTrackerUrl', u+'matomo.php']); _paq.push(['setSiteId', '756']); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); })(); </script>  </body> </html>