Bug #1758683 “combining ROOT files in fixed-order mode” : Bugs : MadGraph5

<!DOCTYPE html> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr"> <head> <base href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+index" /> <meta charset="UTF-8" /> <title>Bug #1758683 “combining ROOT files in fixed-order mode” : Bugs : MadGraph5_aMC@NLO</title> <link rel="apple-touch-icon" sizes="180x180" href="/@@/apple-touch-icon.png?v=2022" /> <link rel="icon" type="image/png" sizes="32x32" href="/@@/favicon-32x32.png?v=2022" /> <link rel="icon" type="image/png" sizes="16x16" href="/@@/favicon-16x16.png?v=2022" /> <link rel="manifest" href="/@@/site.webmanifest?v=2022" /> <link rel="mask-icon" href="/@@/safari-pinned-tab.svg?v=2022" color="#e9531f" /> <link rel="shortcut icon" href="/@@/favicon.ico?v=2022" /> <meta name="msapplication-TileColor" content="#da532c" /> <meta name="msapplication-config" content="/@@/browserconfig.xml?v=2022" /> <meta name="theme-color" content="#ffffff" /> <link rel="canonical" href="https://bugs.launchpad.net/bugs/1758683" /> <link rel="alternate" type="application/atom+xml" href="http://feeds.launchpad.net/bugs/1758683/bug.atom" title="Bug 1758683 Feed" /> <link type="text/css" rel="stylesheet" media="screen, print" href="/+icing/rev78860d903de6d6d7dd5a0ade63efaca45d3467e2/combo.css" /> <meta name="description" content="Dear authors, what I am going to report is not a bug but maybe a feature which can be improved. I am generating a process at NLO such that the last calculation is splitted into about 1500 jobs, and I am filling about 50000 histograms such that each resulting ROOT file has size of 30 MB. In the end of run, combining them in the standard way via combine_root.C has following difficulties: (1) it takes about 12 hours and uses about 100 GB of RAM, so it is pretty not obvious to succeed (2) it eve..." /> <meta property="og:description" content="Dear authors, what I am going to report is not a bug but maybe a feature which can be improved. I am generating a process at NLO such that the last calculation is splitted into about 1500 jobs, and I am filling about 50000 histograms such that each resulting ROOT file has size of 30 MB. In the end of run, combining them in the standard way via combine_root.C has following difficulties: (1) it takes about 12 hours and uses about 100 GB of RAM, so it is pretty not obvious to succeed (2) it eve..." /> <meta property="og:title" content="Bug #1758683 “combining ROOT files in fixed-order mode” : Bugs : MadGraph5_aMC@NLO" /> <meta property="og:type" content="website" /> <meta property="og:image" content="/@@/launchpad-og-image.png" /> <meta property="og:url" content="https://bugs.launchpad.net/bugs/1758683" /> <meta property="og:site_name" content="Launchpad" /> <script type="text/javascript"> var LP = { cache: {}, links: {} }; </script> <script type="text/javascript">var cookie_scope = '; Path=/; Secure; Domain=.launchpad.net';</script> <script type="text/javascript" src="/+combo/rev78860d903de6d6d7dd5a0ade63efaca45d3467e2/?yui/yui/yui-min.js&lp/meta.js&yui/loader/loader-min.js"></script> <script type="text/javascript"> var raw = null; if (LP.devmode) { raw = 'raw'; } YUI.GlobalConfig = { combine: true, comboBase: '/+combo/rev78860d903de6d6d7dd5a0ade63efaca45d3467e2/?', root: 'yui/', filter: raw, debug: false, fetchCSS: false, maxURLLength: 2000, groups: { lp: { combine: true, base: '/+combo/rev78860d903de6d6d7dd5a0ade63efaca45d3467e2/?lp/', comboBase: '/+combo/rev78860d903de6d6d7dd5a0ade63efaca45d3467e2/?', root: 'lp/', // comes from including lp/meta.js modules: LP_MODULES, fetchCSS: false } } }</script> <script type="text/javascript"> // we need this to create a single YUI instance all events and code // talks across. All instances of YUI().use should be based off of // LPJS instead. var LPJS = new YUI(); </script> <script id="base-layout-load-scripts" type="text/javascript"> //<![CDATA[ LPJS.use('base', 'node', 'console', 'event', 'oop', 'lp', 'lp.app.foldables','lp.app.sorttable', 'lp.app.inlinehelp', 'lp.app.links', 'lp.bugs.bugtask_index', 'lp.bugs.subscribers', 'lp.app.ellipsis', 'lp.code.branchmergeproposal.diff', 'lp.views.global', function(Y) { Y.on("domready", function () { var global_view = new Y.lp.views.Global(); global_view.render(); Y.lp.app.sorttable.SortTable.init(); Y.lp.app.inlinehelp.init_help(); Y.lp.activate_collapsibles(); Y.lp.app.foldables.activate(); Y.lp.app.links.check_valid_lp_links(); }); Y.on('lp:context:web_link:changed', function(e) { window.location = e.new_value; }); }); //]]> </script> <script id="base-helper-functions" type="text/javascript"> //<![CDATA[ // This code is pulled from lp.js that needs to be available on every // request. Pulling here to get it outside the scope of the YUI block. function setFocusByName(name) { // Focus the first element matching the given name which can be focused. var nodes = document.getElementsByName(name); var i, node; for (i = 0; i < nodes.length; i++) { node = nodes[i]; if (node.focus) { try { // Trying to focus a hidden element throws an error in IE8. if (node.offsetHeight !== 0) { node.focus(); } } catch (e) { LPJS.use('console', function(Y) { Y.log('In setFocusByName(<' + node.tagName + ' type=' + node.type + '>): ' + e); }); } break; } } } function selectWidget(widget_name, event) { if (event && (event.keyCode === 9 || event.keyCode === 13)) { // Avoid firing if user is tabbing through or simply pressing // enter to submit the form. return; } document.getElementById(widget_name).checked = true; } //]]> </script> <script type="text/javascript" id="available-official-tags-js">var available_official_tags = ["amcatnlo", "madevent", "madfks", "madgraph", "madloop", "madweight"];</script> <script type="text/javascript"> LPJS.use('base', 'node', 'oop', 'event', 'lp.bugs.bugtask_index', 'lp.bugs.subscribers', 'lp.code.branchmergeproposal.diff', 'lp.app.comment', 'lp.services.messages.edit', function(Y) { Y.on('domready', function() { Y.lp.code.branchmergeproposal.diff.connect_diff_links(); Y.lp.bugs.bugtask_index.setup_bugtask_index(); Y.lp.bugs.bugtask_index.setup_bugtask_table(); LP.cache.comment_context = LP.cache.bug; var cl = new Y.lp.app.comment.CommentList(); cl.render(); var sl = new Y.lp.bugs.subscribers.createBugSubscribersLoader({ container_box: '#other-bug-subscribers', subscribers_details_view: '/+bug-portlet-subscribers-details', subscribe_someone_else_link: '.menu-link-addsubscriber' }, window); Y.lp.services.messages.edit.setup(); }); }); </script> <style type="text/css"> /* Align the 'add comment' link to the right of the comment box. */ #add-comment-form textarea { width: 100%; } #add-comment-form { max-width: 60em; padding-bottom: 4em; } #add-comment-form .actions {float: right;} .buglink-summary dd { font-size: 10px; } a#privacy-link:link:hover, a#privacy-link:visited:hover {text-decoration:none;} </style> <style type="text/css"> .yui3-overlay .value label { /* It normally makes sense for form labels to be bold, but since this form consists only of radio buttons, there's nothing but labels so we just get wall-to-wall bold. */ font-weight: normal !important; } </style> </head> <body id="document" itemscope="" itemtype="http://schema.org/WebPage" class="tab-bugs main_side public yui3-skin-sam"> <div class="yui-d0"> <div id="locationbar" class="login-logout"> <div id="logincontrol"><a href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+login">Log in / Register</a></div> </div> <div id="watermark" class="watermark-apps-portlet"> <div> <a href="https://launchpad.net/mg5amcnlo"><img alt="" width="64" height="64" src="https://launchpadlibrarian.net/121184783/64_2.jpg" /></a> </div> <div class="wide"> <h2 id="watermark-heading"><a href="https://launchpad.net/mg5amcnlo">MadGraph5_aMC@NLO</a></h2> </div>  <ul class="facetmenu"> <li class="overview"><a href="https://launchpad.net/mg5amcnlo">Overview</a></li> <li class="branches"><a href="https://code.launchpad.net/mg5amcnlo">Code</a></li> <li class="bugs active"><a href="https://bugs.launchpad.net/mg5amcnlo">Bugs</a></li> <li class="specifications"><a href="https://blueprints.launchpad.net/mg5amcnlo">Blueprints</a></li> <li class="translations"><a href="https://translations.launchpad.net/mg5amcnlo">Translations</a></li> <li class="answers"><a href="https://answers.launchpad.net/mg5amcnlo">Answers</a></li> </ul> </div> <div class="yui-t4"> <div id="maincontent" class="yui-main"> <div class="yui-b" dir="ltr"> <div class="context-publication"> <h1 id="edit-title"> <span class="yui3-editable_text-text ellipsis" style="max-width: 95%;"> combining ROOT files in fixed-order mode </span> </h1> <div id="registration" class="registering"> Bug #1758683 reported by <a href="https://launchpad.net/~zenaiev" class="sprite person">Oleksandr Zenaiev</a> <time title="2018-03-25 15:08:36 UTC" datetime="2018-03-25T15:08:36.775843+00:00">on 2018-03-25</time> </div> </div> <div id="request-notifications"> </div> <div> <div id="bug-is-duplicate"> </div> <div style="float: right;"> <span><a href="/+help-bugs/bug-heat.html" target="help" class="sprite flame">6</a></span> </div> <div class="actions"> <span id="affectsmetoo" style="display: inline">This bug affects 1 person</span> </div> <table id="affected-software" class="listing"> <thead> <tr> <th colspan="2">Affects</th> <th>Status</th> <th>Importance</th> <th>Assigned to</th> <th>Milestone</th> </tr> </thead> <tbody> <tr class="highlight" id="tasksummary2363576"> <td> </td> <td> <span id="bugtarget-picker-tasksummary2363576"> <span class="yui3-activator-data-box"> <a class="sprite product" href="https://bugs.launchpad.net/mg5amcnlo">MadGraph5_aMC@NLO</a> </span> <div class="yui3-activator-message-box yui3-activator-hidden"></div> </span> </td> <td style="width: 20%; vertical-align: middle"> <div class="status-content" style="width: 100%; float: left"> <span style="float: left" class="value statusINPROGRESS">In Progress</span> </div> </td> <td style="width: 15em; vertical-align: middle"> <div class="importance-content" style="width: 100%; float: left"> <span style="float: left" class="value importanceUNDECIDED">Undecided</span> </div> </td> <td style="width:20%; margin: 0; padding: 0; vertical-align: middle; padding-left: 0.5em"> <span id="assignee-picker-tasksummary2363576"> <span class="yui3-activator-data-box"> <a class="sprite person" href="https://launchpad.net/~olivier-mattelaer">Olivier Mattelaer</a> </span> <div class="yui3-activator-message-box yui3-activator-hidden"></div> </span> </td> <td style="width: 20%; vertical-align: middle"> </td> </tr> </tbody> </table> <div id="maincontentsub"> <div class="top-portlet"> <div itemprop="mainContentOfPage" class="report"> <div> <div class="lazr-multiline-edit" id="edit-description"> <div class="clearfix"> <h3>Bug Description</h3> </div> <div class="yui3-editable_text-text"><p>Dear authors,<br /> what I am going to report is not a bug but maybe a feature which can be improved.</p> <p>I am generating a process at NLO such that the last calculation is splitted into about 1500 jobs, and I am filling about 50000 histograms such that each resulting ROOT file has size of 30 MB. In the end of run, combining them in the standard way via combine_root.C has following difficulties:<br /> (1) it takes about 12 hours and uses about 100 GB of RAM, so it is pretty not obvious to succeed<br /> (2) it even does not work out-of-the-box because of the hardcoded limit of 1000 max. files in combine_root.C (of course it was easy to change).</p> <p>Also I've tried the standard 'hadd' ROOT command. It seems to perform much more efficient and gives the identical combined file: with same input it takes ~ 30 minutes and uses 20 GB only.<br /> I am using MG5_aMC_v2_6_0, gcc 5.3 and ROOT 5.34.</p> <p>Is there a reason to use the custom script (combine_root.C) for combining output ROOT files? What do you think, would it make sense to switch to the standard ROOT tool 'hadd' instead?</p> <p>Best regards,<br /> Sasha.</p></div> </div> </div> <div style="margin:-10px 0 20px 5px" class="clearfix"> </div> <div id="bug-tags"> <span id="tags-heading"> </span> <span id="tag-list"> </span> </div> <script type="text/javascript"> LPJS.use('event', 'node', 'lp.bugs.tags_entry', function(Y) { Y.on('domready', function(e) { Y.lp.bugs.tags_entry.setup_tag_entry( available_official_tags); }, window); }); </script> <div class="clearfix"></div> </div> <div id="branches-and-cves"> <div id="bug-branches-container" style="float: left"> </div> <div class="clearfix"></div> </div>  </div> <div> <div class="boardComment"> <div class="boardCommentDetails"> <a href="https://launchpad.net/~olivier-mattelaer" class="sprite person">Olivier Mattelaer (olivier-mattelaer)</a> <time title="2018-03-25 18:35:22 UTC" datetime="2018-03-25T18:35:22.608045+00:00">on 2018-03-25</time> </div> <div class="boardCommentActivity"> <table class="bug-activity"> <tr> <td colspan="2">Changed in mg5amcnlo: </td> </tr> <tr> <td style="text-align: right;"> <b>assignee</b>: </td> <td> nobody → Stefano Frixione (stefano-frixione) </td> </tr> </table> </div> </div> <div xmlns="http://www.w3.org/1999/xhtml" itemscope="" itemtype="http://schema.org/UserComments" class="boardComment editable-message " data-baseurl="/mg5amcnlo/+bug/1758683/comments/1" data-i-can-edit="False"> <div class="boardCommentDetails"> <div class="message-revision-container"> <div class="message-revision-container-header"> <span>Revision history for this message</span> <img src="/+icing/build/overlay/assets/skins/sam/images/close.gif" class="message-revision-close" /> </div> <script type="text/template"> <div class='message-revision-item'> <div class='message-revision-title'> <a class="sprite remove action-icon message-revision-del-btn"> Remove </a> <a class="js-action"> Revision #{revision}, created at {date_created_display} </a> </div> <div class='message-revision-body'>{content}</div> </div> </script> <div class="message-revision-list"></div> </div> <table> <tbody> <tr> <td> <a href="https://launchpad.net/~stefano-frixione" class="sprite person">Stefano Frixione (stefano-frixione)</a> wrote <time itemprop="commentTime" datetime="2018-03-28T08:12:07.757475+00:00" title="2018-03-28 08:12:07 UTC">on 2018-03-28</time><span class="editable-message-last-edit-date">: </span> </td> <td> </td> <td> </td> <td class="bug-comment-index"> <a itemprop="url" href="/mg5amcnlo/+bug/1758683/comments/1"> #1</a> </td> </tr> </tbody> </table> </div> <div class="boardCommentBody"> <div class="editable-message-body"> <div class="comment-text editable-message-text" itemprop="commentText"><p>Dear Sasha,<br /> thanks for reporting this.<br /> Admittedly, that part of the package was never meant to be rock solid, and hasn't been<br /> tested with different root versions, or with large files. Thus, in this case it is not<br /> suprising that it doesn't work well.<br /> In spite of being the original author of that piece of code, I can't remember why hadd<br /> was not deemed a viable option. Maybe it was simply due to my poor understanding of root.<br /> I'll investigate as soon as I'll have some time. In the meanwhile, I have nothing against<br /> you using hadd, but I strongly suggest to test its results against those of the standard<br /> package in a few cases where the latter is supposed to do well, and which are quick to run.<br /> Best, Stefano.</p></div> </div> <div class="editable-message-form" style="display: none"> <textarea style="width: 100%" rows="10">Dear Sasha, thanks for reporting this. Admittedly, that part of the package was never meant to be rock solid, and hasn't been tested with different root versions, or with large files. Thus, in this case it is not suprising that it doesn't work well. In spite of being the original author of that piece of code, I can't remember why hadd was not deemed a viable option. Maybe it was simply due to my poor understanding of root. I'll investigate as soon as I'll have some time. In the meanwhile, I have nothing against you using hadd, but I strongly suggest to test its results against those of the standard package in a few cases where the latter is supposed to do well, and which are quick to run. Best, Stefano. </textarea> <input type="button" value="Update" class="editable-message-update-btn" /> <input type="button" value="Cancel" class="editable-message-cancel-btn" /> </div> </div> </div> <div class="boardComment"> <div class="boardCommentDetails"> <a href="https://launchpad.net/~olivier-mattelaer" class="sprite person">Olivier Mattelaer (olivier-mattelaer)</a> <time title="2018-04-11 15:35:02 UTC" datetime="2018-04-11T15:35:02.904516+00:00">on 2018-04-11</time> </div> <div class="boardCommentActivity"> <table class="bug-activity"> <tr> <td colspan="2">Changed in mg5amcnlo: </td> </tr> <tr> <td style="text-align: right;"> <b>status</b>: </td> <td> New → In Progress </td> </tr> </table> </div> </div> <div xmlns="http://www.w3.org/1999/xhtml" itemscope="" itemtype="http://schema.org/UserComments" class="boardComment editable-message " data-baseurl="/mg5amcnlo/+bug/1758683/comments/2" data-i-can-edit="False"> <div class="boardCommentDetails"> <div class="message-revision-container"> <div class="message-revision-container-header"> <span>Revision history for this message</span> <img src="/+icing/build/overlay/assets/skins/sam/images/close.gif" class="message-revision-close" /> </div> <script type="text/template"> <div class='message-revision-item'> <div class='message-revision-title'> <a class="sprite remove action-icon message-revision-del-btn"> Remove </a> <a class="js-action"> Revision #{revision}, created at {date_created_display} </a> </div> <div class='message-revision-body'>{content}</div> </div> </script> <div class="message-revision-list"></div> </div> <table> <tbody> <tr> <td> <a href="https://launchpad.net/~olivier-mattelaer" class="sprite person">Olivier Mattelaer (olivier-mattelaer)</a> wrote <time itemprop="commentTime" datetime="2019-05-17T09:46:00.039320+00:00" title="2019-05-17 09:46:00 UTC">on 2019-05-17</time><span class="editable-message-last-edit-date">: </span> </td> <td> </td> <td> </td> <td class="bug-comment-index"> <a itemprop="url" href="/mg5amcnlo/+bug/1758683/comments/2"> #2</a> </td> </tr> </tbody> </table> </div> <div class="boardCommentBody"> <div class="editable-message-body"> <div class="comment-text editable-message-text" itemprop="commentText"><p>Hi,</p> <p>I will try to move forward on this.<br /> Would you have a patch (or an example of your changes?)<br /> I have never used root so if you can detail what you did to fix the issue it could be great (otherwise I will wait that someone provide us such clever way of combining histograms)</p> <p>Cheers and thanks,</p> <p>Olivier</p></div> </div> <div class="editable-message-form" style="display: none"> <textarea style="width: 100%" rows="10">Hi, I will try to move forward on this. Would you have a patch (or an example of your changes?) I have never used root so if you can detail what you did to fix the issue it could be great (otherwise I will wait that someone provide us such clever way of combining histograms) Cheers and thanks, Olivier</textarea> <input type="button" value="Update" class="editable-message-update-btn" /> <input type="button" value="Cancel" class="editable-message-cancel-btn" /> </div> </div> <div class="boardCommentActivity"> <table class="bug-activity"> <tr> <td colspan="2">Changed in mg5amcnlo: </td> </tr> <tr> <td style="text-align: right;"> <b>assignee</b>: </td> <td> Stefano Frixione (stefano-frixione) → Olivier Mattelaer (olivier-mattelaer) </td> </tr> </table> </div> </div> <div xmlns="http://www.w3.org/1999/xhtml" itemscope="" itemtype="http://schema.org/UserComments" class="boardComment editable-message " data-baseurl="/mg5amcnlo/+bug/1758683/comments/3" data-i-can-edit="False"> <div class="boardCommentDetails"> <div class="message-revision-container"> <div class="message-revision-container-header"> <span>Revision history for this message</span> <img src="/+icing/build/overlay/assets/skins/sam/images/close.gif" class="message-revision-close" /> </div> <script type="text/template"> <div class='message-revision-item'> <div class='message-revision-title'> <a class="sprite remove action-icon message-revision-del-btn"> Remove </a> <a class="js-action"> Revision #{revision}, created at {date_created_display} </a> </div> <div class='message-revision-body'>{content}</div> </div> </script> <div class="message-revision-list"></div> </div> <table> <tbody> <tr> <td> <a href="https://launchpad.net/~zenaiev" class="sprite person">Oleksandr Zenaiev (zenaiev)</a> wrote <time itemprop="commentTime" datetime="2019-06-29T19:42:43.664678+00:00" title="2019-06-29 19:42:43 UTC">on 2019-06-29</time><span class="editable-message-last-edit-date">: </span> </td> <td> </td> <td> </td> <td class="bug-comment-index"> <a itemprop="url" href="/mg5amcnlo/+bug/1758683/comments/3"> #3</a> </td> </tr> </tbody> </table> </div> <div class="boardCommentBody"> <ul style="margin-bottom: 1em"> <li class="download-attachment"> <a href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+attachment/5274131/+files/diff.txt" class="sprite download-icon">diff.txt</a> <a class="sprite edit action-icon" href="/mg5amcnlo/+bug/1758683/+attachment/5274131">Edit</a> (625 bytes, text/plain) </li> </ul> <div class="editable-message-body"> <div class="comment-text editable-message-text" itemprop="commentText"><p>Hi Olivier,<br /> I did some modification of Template/<wbr />NLO/SubProcesse<wbr />s/combine_<wbr />root.sh, please find attached the diff file.<br /> Let me know if you will have further questions.<br /> Cheers,<br /> Sasha.</p></div> </div> <div class="editable-message-form" style="display: none"> <textarea style="width: 100%" rows="10">Hi Olivier, I did some modification of Template/NLO/SubProcesses/combine_root.sh, please find attached the diff file. Let me know if you will have further questions. Cheers, Sasha.</textarea> <input type="button" value="Update" class="editable-message-update-btn" /> <input type="button" value="Cancel" class="editable-message-cancel-btn" /> </div> </div> </div> <div xmlns="http://www.w3.org/1999/xhtml" itemscope="" itemtype="http://schema.org/UserComments" class="boardComment editable-message " data-baseurl="/mg5amcnlo/+bug/1758683/comments/4" data-i-can-edit="False"> <div class="boardCommentDetails"> <div class="message-revision-container"> <div class="message-revision-container-header"> <span>Revision history for this message</span> <img src="/+icing/build/overlay/assets/skins/sam/images/close.gif" class="message-revision-close" /> </div> <script type="text/template"> <div class='message-revision-item'> <div class='message-revision-title'> <a class="sprite remove action-icon message-revision-del-btn"> Remove </a> <a class="js-action"> Revision #{revision}, created at {date_created_display} </a> </div> <div class='message-revision-body'>{content}</div> </div> </script> <div class="message-revision-list"></div> </div> <table> <tbody> <tr> <td> <a href="https://launchpad.net/~zenaiev" class="sprite person">Oleksandr Zenaiev (zenaiev)</a> wrote <time itemprop="commentTime" datetime="2019-06-29T19:44:50.565648+00:00" title="2019-06-29 19:44:50 UTC">on 2019-06-29</time><span class="editable-message-last-edit-date">: </span> </td> <td> </td> <td> </td> <td class="bug-comment-index"> <a itemprop="url" href="/mg5amcnlo/+bug/1758683/comments/4"> #4</a> </td> </tr> </tbody> </table> </div> <div class="boardCommentBody"> <div class="editable-message-body"> <div class="comment-text editable-message-text" itemprop="commentText"><p>Forgot to mention: this diff is created using MG5_aMC_v2_6_0</p></div> </div> <div class="editable-message-form" style="display: none"> <textarea style="width: 100%" rows="10">Forgot to mention: this diff is created using MG5_aMC_v2_6_0</textarea> <input type="button" value="Update" class="editable-message-update-btn" /> <input type="button" value="Cancel" class="editable-message-cancel-btn" /> </div> </div> </div> <div xmlns="http://www.w3.org/1999/xhtml" itemscope="" itemtype="http://schema.org/UserComments" class="boardComment editable-message " data-baseurl="/mg5amcnlo/+bug/1758683/comments/5" data-i-can-edit="False"> <div class="boardCommentDetails"> <div class="message-revision-container"> <div class="message-revision-container-header"> <span>Revision history for this message</span> <img src="/+icing/build/overlay/assets/skins/sam/images/close.gif" class="message-revision-close" /> </div> <script type="text/template"> <div class='message-revision-item'> <div class='message-revision-title'> <a class="sprite remove action-icon message-revision-del-btn"> Remove </a> <a class="js-action"> Revision #{revision}, created at {date_created_display} </a> </div> <div class='message-revision-body'>{content}</div> </div> </script> <div class="message-revision-list"></div> </div> <table> <tbody> <tr> <td> <a href="https://launchpad.net/~olivier-mattelaer" class="sprite person">Olivier Mattelaer (olivier-mattelaer)</a> wrote <time itemprop="commentTime" datetime="2019-07-03T07:51:07.993107+00:00" title="2019-07-03 07:51:07 UTC">on 2019-07-03</time><span class="editable-message-last-edit-date">: </span> </td> <td> </td> <td> </td> <td class="bug-comment-index"> <a itemprop="url" href="/mg5amcnlo/+bug/1758683/comments/5"> #5</a> </td> </tr> </tbody> </table> </div> <div class="boardCommentBody"> <div class="editable-message-body"> <div class="comment-text editable-message-text" itemprop="commentText"><p>Thanks so much for this.</p> <p>We are reviewing it and will include those changes in the next version.</p> <p>Thanks,</p> <p>Olivier</p></div> </div> <div class="editable-message-form" style="display: none"> <textarea style="width: 100%" rows="10">Thanks so much for this. We are reviewing it and will include those changes in the next version. Thanks, Olivier</textarea> <input type="button" value="Update" class="editable-message-update-btn" /> <input type="button" value="Cancel" class="editable-message-cancel-btn" /> </div> </div> </div> <div style="float: right;"> <a class="menu-link-activitylog" href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+activity">See full activity log</a> </div> <div class="clearfix"></div> <div align="center" id="add-comment-login-first"> To post a comment you must <a href="+login?comments=all">log in</a>. </div> </div> </div> <div> <div id="duplicate-form-container"></div> <div id="privacy-form-container"></div> </div> </div> </div> </div> <div id="side-portlets" class="yui-b side"> <div id="involvement" class="portlet"> <ul class="involvement"> <li class="single"> <a class="sprite bugs" href="/mg5amcnlo/+filebug"> Report a bug </a> </li> </ul> </div> <div id="privacy" class="first portlet public"> <div id="privacy-text"> <span id="information-type-summary" class="sprite public">This report contains <strong id="information-type">Public</strong> information </span>  <div id="information-type-description" style="padding-top: 5px">Everyone can see this information. </div> </div> </div> <div id="portlet-actions" class="portlet vertical"> <ul id="duplicate-actions"> </ul> <ul id="lock-status-actions"> </ul> </div> <div class="portlet vertical" id="portlet-subscription"> <div class="section"> <div id="current_user_subscription" class="False"> <span>You are</span> <a class="menu-link-subscription sprite modify edit" href="/mg5amcnlo/+bug/1758683/+subscribe"> not directly subscribed to this bug's notifications. </a> </div> <div id="sub-unsub-spinner">Subscribing...</div> <ul> <li><a class="menu-link-editsubscriptions sprite modify edit" href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+subscriptions" title="View and change your subscriptions to this bug">Edit bug mail</a></li> </ul> </div> <script type="text/javascript"> LPJS.use('io-base', 'node', 'lp.bugs.bugtask_index.portlets.subscription', function(Y) { Y.on('domready', function() { Y.lp.bugs.bugtask_index.portlets.subscription.initialize(); }); }); </script> </div> <div class="portlet vertical" id="portlet-subscribers"> <h2>Other bug subscribers</h2> <div> <div><a class="menu-link-addsubscriber sprite add" href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+addsubscriber" title="Launchpad will email that person whenever this bugs changes">Subscribe someone else</a></div> </div> <div id="other-bug-subscribers"></div> </div> <div class="portlet" id="portlet-attachments"> <h2>Bug attachments</h2> <ul> <li class="download-attachment"> <a href="https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+attachment/5274131/+files/diff.txt" class="sprite download-icon">diff.txt</a> <a class="sprite edit action-icon" title="Change attachment details" href="/mg5amcnlo/+bug/1758683/+attachment/5274131">Edit</a> </li> </ul> <ul> <li> <a class="sprite add" href="/mg5amcnlo/+bug/1758683/+addcomment">Add attachment</a> </li> </ul> </div> <div class="portlet" id="portlet-watches"> <h2>Remote bug watches</h2> <ul> </ul> <p>Bug watches keep track of this bug in other bug trackers.</p> </div> </div> </div> <div id="footer" class="footer"> <div class="lp-arcana"> <div class="lp-branding"> <a href="https://launchpad.net/"><img src="/@@/launchpad-footer-logo.svg" alt="Launchpad" width="65" height="18" /></a>  •  <a href="https://launchpad.net/+tour">Take the tour</a>  •  <a href="https://help.launchpad.net/">Read the guide</a>   <form id="globalsearch" method="get" accept-charset="UTF-8" action="https://launchpad.net/+search"> <input type="search" id="search-text" name="field.text" /> <input type="image" src="/@@/search" style="vertical-align:5%" alt="Search Launchpad" /> </form> </div> </div> <div class="colophon"> © 2004 <a href="http://canonical.com/">Canonical Ltd.</a>  •  <a href="https://launchpad.net/legal">Terms of use</a>  •  <a href="https://www.ubuntu.com/legal/dataprivacy">Data privacy</a>  •  <a href="/feedback">Contact Launchpad Support</a>  •  <a href="http://blog.launchpad.net/">Blog</a>  •  <a href="https://canonical.com/careers">Careers</a>  •  <a href="https://ubuntu.social/@launchpadstatus">System status</a> <span id="lp-version">  •  78860d9 (<a href="https://dev.launchpad.net/">Get the code!</a>) </span> </div> </div> </div> <script id="json-cache-script">LP.cache = {"related_features": {}, "bug": {"self_link": "https://bugs.launchpad.net/api/devel/bugs/1758683", "web_link": "https://bugs.launchpad.net/bugs/1758683", "resource_type_link": "https://bugs.launchpad.net/api/devel/#bug", "id": 1758683, "private": false, "information_type": "Public", "name": null, "title": "combining ROOT files in fixed-order mode", "description": "Dear authors,\nwhat I am going to report is not a bug but maybe a feature which can be improved.\n\nI am generating a process at NLO such that the last calculation is splitted into about 1500 jobs, and I am filling about 50000 histograms such that each resulting ROOT file has size of 30 MB. In the end of run, combining them in the standard way via combine_root.C has following difficulties:\n(1) it takes about 12 hours and uses about 100 GB of RAM, so it is pretty not obvious to succeed\n(2) it even does not work out-of-the-box because of the hardcoded limit of 1000 max. files in combine_root.C (of course it was easy to change).\n\nAlso I've tried the standard 'hadd' ROOT command. It seems to perform much more efficient and gives the identical combined file: with same input it takes ~ 30 minutes and uses 20 GB only.\nI am using MG5_aMC_v2_6_0, gcc 5.3 and ROOT 5.34.\n\nIs there a reason to use the custom script (combine_root.C) for combining output ROOT files? What do you think, would it make sense to switch to the standard ROOT tool 'hadd' instead?\n\n\nBest regards,\nSasha.", "owner_link": "https://bugs.launchpad.net/api/devel/~zenaiev", "bug_tasks_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/bug_tasks", "duplicate_of_link": null, "date_created": "2018-03-25T15:08:36.775843+00:00", "activity_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/activity", "subscriptions_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/subscriptions", "date_last_updated": "2019-07-03T07:51:08.487386+00:00", "who_made_private_link": null, "date_made_private": null, "heat": 6, "bug_watches_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/bug_watches", "cves_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/cves", "vulnerabilities_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/vulnerabilities", "duplicates_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/duplicates", "attachments_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/attachments", "security_related": false, "latest_patch_uploaded": null, "tags": [], "date_last_message": "2019-07-03T07:51:07.993107+00:00", "number_of_duplicates": 0, "message_count": 6, "users_affected_count": 1, "users_unaffected_count": 0, "users_affected_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/users_affected", "users_unaffected_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/users_unaffected", "users_affected_count_with_dupes": 1, "other_users_affected_count_with_dupes": 1, "users_affected_with_dupes_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/users_affected_with_dupes", "messages_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/messages", "lock_status": "Unlocked", "lock_reason": null, "linked_branches_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/linked_branches", "linked_merge_proposals_collection_link": "https://bugs.launchpad.net/api/devel/bugs/1758683/linked_merge_proposals", "http_etag": "\"c27cf49ed79026f61a916b248a0408847ca420a1-0bed57ef345638b261a7b23c9d6da5dd93664e01\""}, "subscribers_portlet_url_data": {"web_link": "https://bugs.launchpad.net/bugs/1758683", "self_link": "https://bugs.launchpad.net/api/devel/bugs/1758683"}, "total_comments_and_activity": 9, "initial_comment_batch_offset": 41, "first visible_recent_comment": -35, "bugtask_data": {"2363576": {"id": 2363576, "row_id": "tasksummary2363576", "form_row_id": "task2363576", "bugtask_path": "/mg5amcnlo/+bug/1758683", "prefix": "mg5amcnlo", "targetname": "MadGraph5_aMC@NLO", "bug_title": "combining ROOT files in fixed-order mode", "assignee_value": "olivier-mattelaer", "assignee_is_team": false, "assignee_vocabulary": "AllUserTeamsParticipation", "assignee_vocabulary_filters": [], "hide_assignee_team_selection": true, "user_can_unassign": false, "user_can_delete": false, "delete_link": "https://bugs.launchpad.net/mg5amcnlo/+bug/1758683/+delete", "target_is_product": true, "status_widget_items": [{"name": "In Progress", "value": "In Progress", "description": "The assigned person is working on it.\n", "description_css_class": "choice-description", "style": "", "help": "", "disabled": false, "css_class": "statusINPROGRESS"}], "status_value": "In Progress", "importance_widget_items": "[]", "importance_value": "Undecided", "milestone_widget_items": "[]", "milestone_value": null, "user_can_edit_assignee": false, "user_can_edit_milestone": false, "user_can_edit_status": false, "user_can_edit_importance": false}}, "information_type_data": {"PUBLIC": {"value": "PUBLIC", "description": "Everyone can see this information.\n", "name": "Public", "order": 0, "is_private": false, "description_css_class": "choice-description"}, "PUBLICSECURITY": {"value": "PUBLICSECURITY", "description": "Everyone can see this security related information.\n", "name": "Public Security", "order": 1, "is_private": false, "description_css_class": "choice-description"}, "PRIVATESECURITY": {"value": "PRIVATESECURITY", "description": "Only the security group can see this information.\n ", "name": "Private Security", "order": 2, "is_private": true, "description_css_class": "choice-description"}, "USERDATA": {"value": "USERDATA", "description": "Only shared with users permitted to see private user information.\n", "name": "Private", "order": 3, "is_private": true, "description_css_class": "choice-description"}}, "bug_is_private": false, "context": {"self_link": "https://bugs.launchpad.net/api/devel/mg5amcnlo/+bug/1758683", "web_link": "https://bugs.launchpad.net/mg5amcnlo/+bug/1758683", "resource_type_link": "https://bugs.launchpad.net/api/devel/#bug_task", "bug_link": "https://bugs.launchpad.net/api/devel/bugs/1758683", "milestone_link": null, "status": "In Progress", "status_explanation": null, "importance": "Undecided", "importance_explanation": null, "assignee_link": "https://bugs.launchpad.net/api/devel/~olivier-mattelaer", "bug_target_display_name": "MadGraph5_aMC@NLO", "bug_target_name": "mg5amcnlo", "bug_watch_link": null, "date_assigned": "2018-03-25T18:35:22.857105+00:00", "date_created": "2018-03-25T15:08:36.775843+00:00", "date_confirmed": "2018-04-11T15:35:03.128527+00:00", "date_incomplete": null, "date_in_progress": "2018-04-11T15:35:03.128527+00:00", "date_closed": null, "date_left_new": "2018-04-11T15:35:03.128527+00:00", "date_triaged": "2018-04-11T15:35:03.128527+00:00", "date_fix_committed": null, "date_fix_released": null, "date_left_closed": null, "owner_link": "https://bugs.launchpad.net/api/devel/~zenaiev", "target_link": "https://bugs.launchpad.net/api/devel/mg5amcnlo", "title": "Bug #1758683 in MadGraph5_aMC@NLO: \"combining ROOT files in fixed-order mode\"", "related_tasks_collection_link": "https://bugs.launchpad.net/api/devel/mg5amcnlo/+bug/1758683/related_tasks", "is_complete": false, "http_etag": "\"01df3cbc0c8c7a33c999be44c8507938a8446139-cd1684d56c35bebc18281b2722b0689c4783f4cb\""}};</script> </body>  </html>

CINXE.COM

Bug #1758683 “combining ROOT files in fixed-order mode” : Bugs : MadGraph5_aMC@NLO