<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:p="urn:schemas-microsoft-com:office:powerpoint" xmlns:a="urn:schemas-microsoft-com:office:access" xmlns:dt="uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:s="uuid:BDC6E3F0-6DA3-11d1-A2A3-00AA00C14882" xmlns:rs="urn:schemas-microsoft-com:rowset" xmlns:z="#RowsetSchema" xmlns:b="urn:schemas-microsoft-com:office:publisher" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet" xmlns:c="urn:schemas-microsoft-com:office:component:spreadsheet" xmlns:oa="urn:schemas-microsoft-com:office:activation" xmlns:html="http://www.w3.org/TR/REC-html40" xmlns:q="http://schemas.xmlsoap.org/soap/envelope/" xmlns:D="DAV:" xmlns:x2="http://schemas.microsoft.com/office/excel/2003/xml" xmlns:ois="http://schemas.microsoft.com/sharepoint/soap/ois/" xmlns:dir="http://schemas.microsoft.com/sharepoint/soap/directory/" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:dsp="http://schemas.microsoft.com/sharepoint/dsp" xmlns:udc="http://schemas.microsoft.com/data/udc" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:sub="http://schemas.microsoft.com/sharepoint/soap/2002/1/alerts/" xmlns:ec="http://www.w3.org/2001/04/xmlenc#" xmlns:sp="http://schemas.microsoft.com/sharepoint/" xmlns:sps="http://schemas.microsoft.com/sharepoint/soap/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:udcxf="http://schemas.microsoft.com/data/udc/xmlfile" xmlns:wf="http://schemas.microsoft.com/sharepoint/soap/workflow/" xmlns:mver="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns:mrels="http://schemas.openxmlformats.org/package/2006/relationships" xmlns:ex12t="http://schemas.microsoft.com/exchange/services/2006/types" xmlns:ex12m="http://schemas.microsoft.com/exchange/services/2006/messages" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv=Content-Type content="text/html; charset=us-ascii">
<meta name=Generator content="Microsoft Word 12 (filtered medium)">
<style>
<!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;}
@font-face
{font-family:CMR10;}
@font-face
{font-family:CMBX10;}
@font-face
{font-family:CMSS10;}
@font-face
{font-family:CMTI10;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
p.emailquote, li.emailquote, div.emailquote
{mso-style-name:emailquote;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:1.0pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
span.EmailStyle19
{mso-style-type:personal;
font-family:"Calibri","sans-serif";
color:#1F497D;}
span.EmailStyle20
{mso-style-type:personal;
font-family:"Calibri","sans-serif";
color:#1F497D;}
span.EmailStyle21
{mso-style-type:personal-reply;
font-family:"Calibri","sans-serif";
color:#1F497D;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.25in 1.0in 1.25in;}
div.Section1
{page:Section1;}
/* List Definitions */
@list l0
{mso-list-id:1536773038;
mso-list-type:hybrid;
mso-list-template-ids:-175185672 685036460 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri","sans-serif";
mso-fareast-font-family:Calibri;
mso-bidi-font-family:Arial;}
@list l0:level2
{mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level3
{mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level4
{mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level5
{mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level6
{mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level7
{mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level8
{mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level9
{mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
-->
</style>
<!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang=EN-US link=blue vlink=purple>
<div class=Section1>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>I’ve added the wiki page for the send buffer access
proposal, please review.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><a
href="http://svn.mpi-forum.org/trac/mpi-forum-web/wiki/SendBufferAccess">http://svn.mpi-forum.org/trac/mpi-forum-web/wiki/SendBufferAccess</a><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Thanks,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>.Erez<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'> mpi-buff_access-bounces@lists.mpi-forum.org
[mailto:mpi-buff_access-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Erez
Haba<br>
<b>Sent:</b> Wednesday, February 27, 2008 1:49 PM<br>
<b>To:</b> mpi-buff_access@lists.mpi-forum.org<br>
<b>Subject:</b> Re: [Mpi-buff_access] MPI Send Buffer Access - discussion<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>For those who were not able to read the pptx slides here the ppt
version.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'> Erez Haba <br>
<b>Sent:</b> Tuesday, February 26, 2008 9:02 AM<br>
<b>To:</b> 'mpi-buff_access@lists.mpi-forum.org'<br>
<b>Subject:</b> MPI Send Buffer Access - discussion<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Hello gents,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>This email thread is about removing the send buffer access
restrictions from the MPI standard. I’ve attached the original proposal
(below) and the slides I presented in the Jan forum meeting.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Let me know if you think that we need to add or remove content.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>I think that this proposal is good for the March forum meeting
with few things we need to complete.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>TODO list:<o:p></o:p></span></p>
<p class=MsoListParagraph style='text-indent:-.25in;mso-list:l0 level1 lfo2'><![if !supportLists]><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><span
style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'>
</span></span></span><![endif]><span dir=LTR></span><span style='font-size:
11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'>Post this proposal on
the wiki pages<o:p></o:p></span></p>
<p class=MsoListParagraph style='text-indent:-.25in;mso-list:l0 level1 lfo2'><![if !supportLists]><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><span
style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'>
</span></span></span><![endif]><span dir=LTR></span><span style='font-size:
11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'>Address any cons not
addressed in this proposal. Do you know of any?<o:p></o:p></span></p>
<p class=MsoListParagraph style='text-indent:-.25in;mso-list:l0 level1 lfo2'><![if !supportLists]><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><span
style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'>
</span></span></span><![endif]><span dir=LTR></span><span style='font-size:
11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'>Add Examples from real
scientific applications that would benefit from this proposal.<o:p></o:p></span></p>
<p class=MsoListParagraph><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>If you have any example please bring it forward.<o:p></o:p></span></p>
<p class=MsoListParagraph style='text-indent:-.25in;mso-list:l0 level1 lfo2'><![if !supportLists]><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><span
style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'>
</span></span></span><![endif]><span dir=LTR></span><span style='font-size:
11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'>Please review the
proposal against the MPI spec to verify if any additional modifications to the
spec are required.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Please let me know if you think we need a conf call for this
proposal.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Thanks,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>.Erez<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>
owner-mpi-21@mpi-forum.org [mailto:owner-mpi-21@mpi-forum.org] <b>On Behalf Of </b>Erez
Haba<br>
<b>Sent:</b> Monday, December 17, 2007 10:14 AM<br>
<b>To:</b> mpi-21@mpi-forum.org<br>
<b>Subject:</b> [mpi-21] Proposal EH1: Send buffer access<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
is the proposal to remove the access restriction on the user send buffers. I
think it make sense to apply this to the 2.1 document however I’m okay
with this being discussed for 2.2.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Thanks,<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>.Erez<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Background:</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>MPI
1.1 standard prohibits users from accessing for read their send buffer until
the send operation completes. Be it access in the same thread in case of an
async send operation or by another thread in the case of blocking send
operation. The rational in the MPI 1.1 standard was to enable the performance
for DMA engine that is not cache-coherent with the main processor.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Proposal:</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Remove
the access restriction on the send buffers.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Rational:</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
restriction is counter intuitive for many programmers. It leads to programs
that are not compliant with the spec, unaware of this limitation. This
limitation prohibits a common usage of the async send API’s, and or the
blocking sends.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>For
example the following code sequence is not compliant with the spec.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>void* my_buffer;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>…</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>MPI_Isend(my_buffer, 100, MPI_CHAR, 3, 0, MPI_COMM_WORLD,
&request1);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>MPI_Isend(my_buffer, 100, MPI_CHAR, 4, 0, MPI_COMM_WORLD,
&request2);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
email thread discussing the cache-coherency machines limitation concluded that
it’s a non issue for these machines. The application should resolve the
multi-threaded non cache-coherency issues anyhow for sending data.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Cons:</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
change will render implementations that modify the user buffer in-place (like
byte-swap) invalid.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Proposed
changes to the MPI document:</span></b><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi1-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
3.4 Page 27 Line 17</span></i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Change:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>The send
call described in Section 3.2.1 is </span><span style='font-size:10.0pt;
font-family:CMBX10'>blocking</span><span style='font-size:10.0pt;font-family:
CMR10'>: it does not return until the message</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>data and
envelope have been safely stored away so that the sender is free to <span
style='background:yellow'>access and</span></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>overwrite</span><span style='font-size:10.0pt;font-family:CMR10'> the
send buffer.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>To:
(change “access and overwrite” to “modify”)<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>The send
call described in Section 3.2.1 is </span><span style='font-size:10.0pt;
font-family:CMBX10'>blocking</span><span style='font-size:10.0pt;font-family:
CMR10'>: it does not return until the message</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>data and
envelope have been safely stored away so that the sender is free to </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>modify</span><span style='font-size:10.0pt;font-family:CMR10'> the send
buffer.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi1-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
3.4 Page 29 Line 48</span></i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Change:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>In a
multi-threaded implementation of </span><span style='font-size:10.0pt;
font-family:CMSS10'>MPI</span><span style='font-size:10.0pt;font-family:CMR10'>,
the system may de-schedule a thread that</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>is blocked
on a send or receive operation, and schedule another thread for execution in
the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>same
address space. In such a case it is the user’s responsibility not to <span
style='background:yellow'>access or</span> modify a</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>communication
buffer until the communication completes. Otherwise, the outcome of the</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>computation
is undefined.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>To:
(remove “access or”)<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>In a
multi-threaded implementation of </span><span style='font-size:10.0pt;
font-family:CMSS10'>MPI</span><span style='font-size:10.0pt;font-family:CMR10'>,
the system may de-schedule a thread that</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>is blocked
on a send or receive operation, and schedule another thread for execution in
the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>same
address space. In such a case it is the user’s responsibility not to
modify a</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>communication
buffer until the communication completes. Otherwise, the outcome of the</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>computation
is undefined.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi1-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
3.4 Page 30 Lines 4-9</span></i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Remove:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:CMTI10;
background:yellow'>Rationale. </span></i><span style='font-size:10.0pt;
font-family:CMR10;background:yellow'>We prohibit read accesses to a send buffer
while it is being used, even</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>though the send operation is not supposed to alter the content of this
buffer. This</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>may seem more stringent than necessary, but the additional restriction
causes little</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>loss of functionality and allows better performance on some systems
— consider the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>case where data transfer is done by a DMA engine that is not
cache-coherent with the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>main processor. (</span><i><span style='font-size:10.0pt;font-family:
CMTI10;background:yellow'>End of rationale.</span></i><span style='font-size:
10.0pt;font-family:CMR10;background:yellow'>)</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi1-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
3.9 Page 58 Line 11</span></i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Change:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>If the
request is for a send with ready mode, then a matching receive should be posted</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>before the
call is made. The communication buffer should not be <span style='background:
yellow'>accessed</span> after the call,</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>and until
the operation completes.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>To:
(change “accessed” to “modified”)<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>If the
request is for a send with ready mode, then a matching receive should be posted</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>before the
call is made. The communication buffer should not be <span style='background:
yellow'>modified</span> after the call,</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>and until
the operation completes.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi1-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
4.1 Page 94 Line 12</span></i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Change:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>Collective
routine calls can (but are not required to) return as soon as their participation</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>in the
collective communication is complete. The completion of a call indicates that
the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>caller is
now free to <span style='background:yellow'>access</span> locations in the
communication buffer.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>To:
(change “access” to “modify”)<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>Collective
routine calls can (but are not required to) return as soon as their
participation</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>in the
collective communication is complete. The completion of a call indicates that
the</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10'>caller is
now free to <span style='background:yellow'>modify</span> locations in the
communication buffer.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>mpi2-report.pdf:</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Section
6.3 Page 112 Line 20-31</span></i><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Remove:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:CMTI10;
background:yellow'>Rationale. </span></i><span style='font-size:10.0pt;
font-family:CMR10;background:yellow'>The rule above is more lenient than for
message passing, where we do</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>not allow two concurrent sends, with overlapping send buffers. Here, we
allow two</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>concurrent puts with overlapping send buffers. The reasons for this
relaxation are</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>1. Users do not like that restriction, which is not very natural (it
prohibits concurrent </span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>reads).</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>2. Weakening the rule does not prevent efficient implementation, as far
as we know.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>3. Weakening the rule is important for performance of </span><span
style='font-size:10.0pt;font-family:CMSS10;background:yellow'>RMA</span><span
style='font-size:10.0pt;font-family:CMR10;background:yellow'>: we want to
associate</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>one synchronization call with as many </span><span style='font-size:
10.0pt;font-family:CMSS10;background:yellow'>RMA </span><span style='font-size:
10.0pt;font-family:CMR10;background:yellow'>operations is possible. If puts
from</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>overlapping buffers cannot be concurrent, then we need to needlessly
add synchronization</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>points in the code.</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:CMR10;background:
yellow'>(</span><i><span style='font-size:10.0pt;font-family:CMTI10;background:
yellow'>End of rationale.</span></i><span style='font-size:10.0pt;font-family:
CMR10;background:yellow'>)</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
</div>
</body>
</html>