CINXE.COM

<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /><title>LKML: Roland Dreier: [PATCH][1/27] IB/mthca: map MPT/MTT context in mem-free mode</title><link href="/css/message.css" rel="stylesheet" type="text/css" /><link href="/css/wrap.css" rel="alternate stylesheet" type="text/css" title="wrap" /><link href="/css/nowrap.css" rel="stylesheet" type="text/css" title="nowrap" /><link href="/favicon.ico" rel="shortcut icon" /><script src="/js/simple-calendar.js" type="text/javascript"></script><script src="/js/styleswitcher.js" type="text/javascript"></script><link rel="alternate" type="application/rss+xml" title="lkml.org : last 100 messages" href="/rss.php" /><link rel="alternate" type="application/rss+xml" title="lkml.org : last messages by Roland Dreier" href="/groupie.php?aid=3215" /><script> var _paq = window._paq = window._paq || []; /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ _paq.push(["setDoNotTrack", true]); _paq.push(["disableCookies"]); _paq.push(['trackPageView']); _paq.push(['enableLinkTracking']); (function() { var u="//m.lkml.org/"; _paq.push(['setTrackerUrl', u+'matomo.php']); _paq.push(['setSiteId', '1']); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); })(); </script></head><body onload="es.jasper.simpleCalendar.init();" itemscope="itemscope" itemtype="http://schema.org/BlogPosting"><table border="0" cellpadding="0" cellspacing="0"><tr><td width="180" align="center"><a href="/"><img style="border:0;width:135px;height:32px" src="/images/toprowlk.gif" alt="lkml.org" /></a></td><td width="32">聽</td><td class="nb"><div><a class="nb" href="/lkml"> [lkml]</a> 聽 <a class="nb" href="/lkml/2005"> [2005]</a> 聽 <a class="nb" href="/lkml/2005/4"> [Apr]</a> 聽 <a class="nb" href="/lkml/2005/4/1"> [1]</a> 聽 <a class="nb" href="/lkml/last100"> [last100]</a> 聽 <a href="/rss.php"><img src="/images/rss-or.gif" border="0" alt="RSS Feed" /></a></div><div>Views: <a href="#" class="nowrap" onclick="setActiveStyleSheet('wrap');return false;">[wrap]</a><a href="#" class="wrap" onclick="setActiveStyleSheet('nowrap');return false;">[no wrap]</a> 聽 <a class="nb" href="/lkml/mheaders/2005/4/1/224" onclick="this.href='/lkml/headers'+'/2005/4/1/224';">[headers]</a>聽 <a href="/lkml/bounce/2005/4/1/224">[forward]</a>聽 </div></td><td width="32">聽</td></tr><tr><td valign="top"><div class="es-jasper-simpleCalendar" baseurl="/lkml/"></div><div class="threadlist">Messages in this thread</div><ul class="threadlist"><li class="root"><a href="/lkml/2005/4/1/224">First message in thread</a></li><li class="origin"><a href="/lkml/2005/4/1/223">Roland Dreier</a><ul><li><a href="/lkml/2005/4/1/223">Roland Dreier</a><ul><li><a href="/lkml/2005/4/1/257">Roland Dreier</a><ul><li><a href="/lkml/2005/4/1/256">Roland Dreier</a><ul><li><a href="/lkml/2005/4/1/258">Roland Dreier</a></li></ul></li></ul></li></ul></li></ul></li></ul><div class="threadlist">Patch in this message</div><ul class="threadlist"><li><a href="/lkml/diff/2005/4/1/224/1">Get diff 1</a></li></ul></td><td width="32" rowspan="2" class="c" valign="top"><img src="/images/icornerl.gif" width="32" height="32" alt="/" /></td><td class="c" rowspan="2" valign="top" style="padding-top: 1em"><table><tr><td><table><tr><td class="lp">Subject</td><td class="rp" itemprop="name">[PATCH][1/27] IB/mthca: map MPT/MTT context in mem-free mode</td></tr><tr><td class="lp">Date</td><td class="rp" itemprop="datePublished">Fri, 1 Apr 2005 12:49:52 -0800</td></tr><tr><td class="lp">From</td><td class="rp" itemprop="author">Roland Dreier <></td></tr></table></td><td></td></tr></table><pre itemprop="articleBody">In mem-free mode, when allocating memory regions, make sure that the HCA has context memory mapped to cover the virtual space used for the MPT and MTTs being used. Signed-off-by: Roland Dreier <roland@topspin.com> --- linux-export.orig/drivers/infiniband/hw/mthca/mthca_main.c 2005-03-31 19:06:51.000000000 -0800 +++ linux-export/drivers/infiniband/hw/mthca/mthca_main.c 2005-04-01 12:38:19.884644268 -0800 @@ -390,7 +390,7 @@ } mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base, - init_hca->mtt_seg_sz, + dev_lim->mtt_seg_sz, mdev->limits.num_mtt_segs, mdev->limits.reserved_mtts, 1); if (!mdev->mr_table.mtt_table) { --- linux-export.orig/drivers/infiniband/hw/mthca/mthca_memfree.c 2005-03-31 19:06:42.000000000 -0800 +++ linux-export/drivers/infiniband/hw/mthca/mthca_memfree.c 2005-04-01 12:38:19.911638409 -0800 @@ -192,6 +192,38 @@ up(&table->mutex); } +int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, + int start, int end) +{ + int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; + int i, err; + + for (i = start; i <= end; i += inc) { + err = mthca_table_get(dev, table, i); + if (err) + goto fail; + } + + return 0; + +fail: + while (i > start) { + i -= inc; + mthca_table_put(dev, table, i); + } + + return err; +} + +void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, + int start, int end) +{ + int i; + + for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) + mthca_table_put(dev, table, i); +} + struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, u64 virt, int obj_size, int nobj, int reserved, --- linux-export.orig/drivers/infiniband/hw/mthca/mthca_memfree.h 2005-03-31 19:06:56.000000000 -0800 +++ linux-export/drivers/infiniband/hw/mthca/mthca_memfree.h 2005-04-01 12:38:19.895641881 -0800 @@ -85,6 +85,10 @@ void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table); int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); +int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, + int start, int end); +void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, + int start, int end); static inline void mthca_icm_first(struct mthca_icm *icm, struct mthca_icm_iter *iter) --- linux-export.orig/drivers/infiniband/hw/mthca/mthca_mr.c 2005-03-31 19:07:06.000000000 -0800 +++ linux-export/drivers/infiniband/hw/mthca/mthca_mr.c 2005-04-01 12:38:19.903640145 -0800 @@ -38,6 +38,7 @@ #include "mthca_dev.h" #include "mthca_cmd.h" +#include "mthca_memfree.h" /* * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. @@ -71,7 +72,7 @@ * through the bitmaps) */ -static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order) +static u32 __mthca_alloc_mtt(struct mthca_dev *dev, int order) { int o; int m; @@ -105,7 +106,7 @@ return seg; } -static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order) +static void __mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order) { seg >>= order; @@ -122,6 +123,32 @@ spin_unlock(&dev->mr_table.mpt_alloc.lock); } +static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order) +{ + u32 seg = __mthca_alloc_mtt(dev, order); + + if (seg == -1) + return -1; + + if (dev->hca_type == ARBEL_NATIVE) + if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, + seg + (1 << order) - 1)) { + __mthca_free_mtt(dev, seg, order); + seg = -1; + } + + return seg; +} + +static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order) +{ + __mthca_free_mtt(dev, seg, order); + + if (dev->hca_type == ARBEL_NATIVE) + mthca_table_put_range(dev, dev->mr_table.mtt_table, seg, + seg + (1 << order) - 1); +} + static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) { if (dev->hca_type == ARBEL_NATIVE) @@ -141,7 +168,7 @@ int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, u32 access, struct mthca_mr *mr) { - void *mailbox; + void *mailbox = NULL; struct mthca_mpt_entry *mpt_entry; u32 key; int err; @@ -155,11 +182,17 @@ return -ENOMEM; mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); + if (dev->hca_type == ARBEL_NATIVE) { + err = mthca_table_get(dev, dev->mr_table.mpt_table, key); + if (err) + goto err_out_mpt_free; + } + mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA, GFP_KERNEL); if (!mailbox) { - mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); - return -ENOMEM; + err = -ENOMEM; + goto err_out_table; } mpt_entry = MAILBOX_ALIGN(mailbox); @@ -180,16 +213,27 @@ err = mthca_SW2HW_MPT(dev, mpt_entry, key & (dev->limits.num_mpts - 1), &status); - if (err) + if (err) { mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); - else if (status) { + goto err_out_table; + } else if (status) { mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", status); err = -EINVAL; + goto err_out_table; } kfree(mailbox); return err; + +err_out_table: + if (dev->hca_type == ARBEL_NATIVE) + mthca_table_put(dev, dev->mr_table.mpt_table, key); + +err_out_mpt_free: + mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); + kfree(mailbox); + return err; } int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, @@ -213,6 +257,12 @@ return -ENOMEM; mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); + if (dev->hca_type == ARBEL_NATIVE) { + err = mthca_table_get(dev, dev->mr_table.mpt_table, key); + if (err) + goto err_out_mpt_free; + } + for (i = dev->limits.mtt_seg_size / 8, mr->order = 0; i < list_len; i <<= 1, ++mr->order) @@ -220,7 +270,7 @@ mr->first_seg = mthca_alloc_mtt(dev, mr->order); if (mr->first_seg == -1) - goto err_out_mpt_free; + goto err_out_table; /* * If list_len is odd, we add one more dummy entry for @@ -307,13 +357,17 @@ kfree(mailbox); return err; - err_out_mailbox_free: +err_out_mailbox_free: kfree(mailbox); - err_out_free_mtt: +err_out_free_mtt: mthca_free_mtt(dev, mr->first_seg, mr->order); - err_out_mpt_free: +err_out_table: + if (dev->hca_type == ARBEL_NATIVE) + mthca_table_put(dev, dev->mr_table.mpt_table, key); + +err_out_mpt_free: mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); return err; } @@ -338,6 +392,9 @@ if (mr->order >= 0) mthca_free_mtt(dev, mr->first_seg, mr->order); + if (dev->hca_type == ARBEL_NATIVE) + mthca_table_put(dev, dev->mr_table.mpt_table, + key_to_hw_index(dev, mr->ibmr.lkey)); mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey)); } - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at <a href="http://vger.kernel.org/majordomo-info.html">http://vger.kernel.org/majordomo-info.html</a> Please read the FAQ at <a href="http://www.tux.org/lkml/">http://www.tux.org/lkml/</a> </pre></td><td width="32" rowspan="2" class="c" valign="top"><img src="/images/icornerr.gif" width="32" height="32" alt="\" /></td></tr><tr><td align="right" valign="bottom"> 聽 </td></tr><tr><td align="right" valign="bottom">聽</td><td class="c" valign="bottom" style="padding-bottom: 0px"><img src="/images/bcornerl.gif" width="32" height="32" alt="\" /></td><td class="c">聽</td><td class="c" valign="bottom" style="padding-bottom: 0px"><img src="/images/bcornerr.gif" width="32" height="32" alt="/" /></td></tr><tr><td align="right" valign="top" colspan="2"> 聽 </td><td class="lm">Last update: 2005-04-06 13:31 聽聽 [from the cache] 漏2003-2020 <a href="http://blog.jasper.es/">Jasper Spaans</a>|hosted at <a href="https://www.digitalocean.com/?refcode=9a8e99d24cf9">Digital Ocean</a> and my Meterkast|<a href="http://blog.jasper.es/categories.html#lkml-ref">Read the blog</a></td><td>聽</td></tr></table><script language="javascript" src="/js/styleswitcher.js" type="text/javascript"></script></body></html>