<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:p="urn:schemas-microsoft-com:office:powerpoint" xmlns:a="urn:schemas-microsoft-com:office:access" xmlns:dt="uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:s="uuid:BDC6E3F0-6DA3-11d1-A2A3-00AA00C14882" xmlns:rs="urn:schemas-microsoft-com:rowset" xmlns:z="#RowsetSchema" xmlns:b="urn:schemas-microsoft-com:office:publisher" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet" xmlns:c="urn:schemas-microsoft-com:office:component:spreadsheet" xmlns:odc="urn:schemas-microsoft-com:office:odc" xmlns:oa="urn:schemas-microsoft-com:office:activation" xmlns:html="http://www.w3.org/TR/REC-html40" xmlns:q="http://schemas.xmlsoap.org/soap/envelope/" xmlns:D="DAV:" xmlns:mt="http://schemas.microsoft.com/sharepoint/soap/meetings/" xmlns:x2="http://schemas.microsoft.com/office/excel/2003/xml" xmlns:ois="http://schemas.microsoft.com/sharepoint/soap/ois/" xmlns:dir="http://schemas.microsoft.com/sharepoint/soap/directory/" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:dsp="http://schemas.microsoft.com/sharepoint/dsp" xmlns:udc="http://schemas.microsoft.com/data/udc" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:sub="http://schemas.microsoft.com/sharepoint/soap/2002/1/alerts/" xmlns:ec="http://www.w3.org/2001/04/xmlenc#" xmlns:sp="http://schemas.microsoft.com/sharepoint/" xmlns:sps="http://schemas.microsoft.com/sharepoint/soap/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:udcs="http://schemas.microsoft.com/data/udc/soap" xmlns:udcxf="http://schemas.microsoft.com/data/udc/xmlfile" xmlns:udcp2p="http://schemas.microsoft.com/data/udc/parttopart" xmlns:wf="http://schemas.microsoft.com/sharepoint/soap/workflow/" xmlns:dsss="http://schemas.microsoft.com/office/2006/digsig-setup" xmlns:dssi="http://schemas.microsoft.com/office/2006/digsig" xmlns:mdssi="http://schemas.openxmlformats.org/package/2006/digital-signature" xmlns:mver="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns:mrels="http://schemas.openxmlformats.org/package/2006/relationships" xmlns:spwp="http://microsoft.com/sharepoint/webpartpages" xmlns:ex12t="http://schemas.microsoft.com/exchange/services/2006/types" xmlns:ex12m="http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:pptsl="http://schemas.microsoft.com/sharepoint/soap/SlideLibrary/" xmlns:spsl="http://microsoft.com/webservices/SharePointPortalServer/PublishedLinksService" xmlns:Z="urn:schemas-microsoft-com:" xmlns:st="" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv=Content-Type content="text/html; charset=us-ascii">
<meta name=Generator content="Microsoft Word 12 (filtered medium)">
<!--[if !mso]>
<style>
v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style>
<![endif]-->
<style>
<!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Cambria;
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
pre
{mso-style-priority:99;
mso-style-link:"HTML Preformatted Char";
margin:0in;
margin-bottom:.0001pt;
font-size:10.0pt;
font-family:"Courier New";}
p.MsoAcetate, li.MsoAcetate, div.MsoAcetate
{mso-style-priority:99;
mso-style-link:"Balloon Text Char";
margin:0in;
margin-bottom:.0001pt;
font-size:8.0pt;
font-family:"Tahoma","sans-serif";}
span.HTMLPreformattedChar
{mso-style-name:"HTML Preformatted Char";
mso-style-priority:99;
mso-style-link:"HTML Preformatted";
font-family:Consolas;}
span.BalloonTextChar
{mso-style-name:"Balloon Text Char";
mso-style-priority:99;
mso-style-link:"Balloon Text";
font-family:"Tahoma","sans-serif";}
p.emailquote, li.emailquote, div.emailquote
{mso-style-name:emailquote;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:1.0pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
span.EmailStyle22
{mso-style-type:personal;
font-family:"Calibri","sans-serif";
color:#1F497D;}
span.EmailStyle23
{mso-style-type:personal;
font-family:"Calibri","sans-serif";
color:#1F497D;}
span.EmailStyle24
{mso-style-type:personal-reply;
font-family:"Calibri","sans-serif";
color:#1F497D;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.Section1
{page:Section1;}
-->
</style>
<!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang=EN-US link=blue vlink=purple>
<div class=Section1>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Thanks Alexander, please see inline…<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Supalov,
Alexander<br>
<b>Sent:</b> Wednesday, February 25, 2009 2:20 PM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> Re: [Mpi3-ft] MPI Fault Tolerance scenarios<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>Thanks. What error handler will be in effect when the
MPI_Comm_Irepair is called? If it's MPI_ERRORS_ARE_FATAL, an error will
terminate the application. If it's MPI_ERRORS_RETURN, one will either have to
make MPI_Comm_Irepair and some other proposed FT calls an exception
compared to the rest of the MPI standard, or explicitly set the error handler
on the respective comm to MPI_ERRORS_RETURN.</span><span style='font-size:10.0pt;
font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] This server side code sets the error handler to
MPI_ERROR_RETURNS. I don’t see the need for any special case in this API,
or am I missing something?<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>In generate if your application is FT you should not have
MPI_ERRORS_ARE_FATAL as your error handler as It beats the purpose of FT app.<o:p></o:p></span></i></b></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>And if one anyway has to set the error handler to something else
that MPI_ERRORS_ARE_FATAL, why not delegating this whole comm restoration stuff
to the error handler? The handler could call the MPI_Comm_Irepair or anything
else required, and either return control to the application on success, or
abort it using MPI_Abort on the respective communicator.</span><span
style='font-size:10.0pt;font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] that is possible to handle to error in your error
handler. However this makes the error handler code synchronous and would block
the app until the failed process is restarted. This might take some time, which
is why I used the async version of repair in the master’s code, hence
using Irepair and not repair.<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Two more things though, one I don’t want to abort in the
case that I could not respawn the worker. I rather just shrink the number of
workers. I don’t think it’s possible to do that in an error
handler. Second, I’d like to reassign the work to another worker. I can’t
do that if the error is detected when the receive is posted, and anyhow the
error handler might be called twice in this case, once during the send and once
during the receive. I think that would have complicated the code to distinguish
if a repair was already invoked.</span></i></b><span style='font-size:11.0pt;
font-family:"Calibri","sans-serif";color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>By defining several standard error handlers for various fault tolerance
policies we could implement what Bronis was proposing I guess. Note that these
error handlers could either use the proposed low level API, or do something
implementation specific do achieve their stated goal.</span><span
style='font-size:10.0pt;font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] I think that it depends very much on the application. For
some it might work, but for others I don’t think it would.<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>If you take this simple scenario code, I don’t think it
would be simple as you suggest. The application need to be notified when a
repair failed and need to decide to continue and run with less workers. I think
that would be hard to accomplish with pre-canned solutions.</span></i></b><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>Also, different implementations could have different sets of
provided standard error handlers, and thus vary the level of the fault
tolerance support provided, which is what I'd be looking for. This includes
providing no FT support at all by defining no special FT error handlers, thus
making the FT stuff completely orthogonal to the rest of the standard.</span><span
style='font-size:10.0pt;font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] I think that at the moment we’re trying to understand
the primitives required for FT, rather that jump into defining everyone favorite
programming model. By being explicit we understand the primitives and constructs
needed. Once we done that, we can go and recan the model into some fancy
API/set of fags/…</span></i></b><span style='font-size:11.0pt;font-family:
"Calibri","sans-serif";color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>And finally, this would indeed make FT transparent to the
application. The proposed use of the low level API litters the application code
with error </span><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>checks and reaction to every possible error condition in the
application code. This does not seem to be the right way.</span><span
style='font-size:10.0pt;font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] I think we had the general agreement in the FT
workgroup that we can NOT make FT transparent, nether this is a goal for the
WG.<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>In any case, shouldn’t you check for returned error code
in your app anyway? Otherwise you run into corrupted data bugs. (or you don’t
care?)<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>I can think of a nicer programming model where instead of
returned errors you get exception and you have your error handling routines dealing
with the errors… oh wait we have that with the C++ bindings… </span></i></b><b><i><span
style='font-size:11.0pt;font-family:Wingdings;color:#1F497D'>J</span></i></b><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>The proposed error handling approach is backward compatible,
flexible, elegant, and simple. Thoughts?</span><span style='font-size:10.0pt;
font-family:"Arial","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>[erezh] do you mean you get FT for free?<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>The current explicit API’s are backward compatible. They
have zero affect on existing applications. I think that the model we are
driving toward is flexible, elegant and simple… plus preferment. You pay
extra only when you care about FT and even that can be amortized.<o:p></o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></i></b></p>
<p class=MsoNormal><b><i><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>I think that at the moment you only see a fraction of the model.
With few more scenarios I think that you’ll be able to see the complexity
of the problem, plus the beauty of the solution we’re working towards.</span></i></b><span
style='font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D'><o:p></o:p></span></p>
<p class=MsoNormal><o:p> </o:p></p>
<div class=MsoNormal align=center style='text-align:center'>
<hr size=2 width="100%" align=center>
</div>
<p class=MsoNormal style='margin-bottom:12.0pt'><b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'>From:</span></b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Erez Haba<br>
<b>Sent:</b> Wednesday, February 25, 2009 5:45 PM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> Re: [Mpi3-ft] MPI Fault Tolerance scenarios</span><o:p></o:p></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>MPI_Comm_Irepair<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Supalov,
Alexander<br>
<b>Sent:</b> Wednesday, February 25, 2009 8:40 AM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> Re: [Mpi3-ft] MPI Fault Tolerance scenarios<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif";
color:blue'>Thanks. What call is taking care of respawning? This is not crystal
clear from the scenario I'm afraid.</span><o:p></o:p></p>
<p class=MsoNormal><o:p> </o:p></p>
<div class=MsoNormal align=center style='text-align:center'>
<hr size=2 width="100%" align=center>
</div>
<p class=MsoNormal style='margin-bottom:12.0pt'><b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'>From:</span></b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Erez Haba<br>
<b>Sent:</b> Wednesday, February 25, 2009 5:35 PM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> Re: [Mpi3-ft] MPI Fault Tolerance scenarios</span><o:p></o:p></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Hi Alexander,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>As explained in the intro section, we are building the scenarios
bottom up to create solid foundations for the FT architecture. However you are
right and the scenarios will expand to use collectives and other communicators,
just no in this first scenario.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>This scenario is very very simple, and very very basic, to show
the absolute minimum required for process FT. It only uses point-to-point
communication over com world. The workers are stateless and can be respawned
without any need for state recovery.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Thanks,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>.Erez<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>
mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Supalov, Alexander<br>
<b>Sent:</b> Wednesday, February 25, 2009 6:25 AM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> Re: [Mpi3-ft] MPI Fault Tolerance scenarios<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>Dear
Erez,</span><o:p></o:p></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>Thank
you. A couple of questions:</span><o:p></o:p></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>1.
You seem to restrict communication to pt2pt only. Why? A Bcast upfront could be
useful, for one.</span><o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>2.
I can imagine more complicated communicator combinations than only
MPI_COMM_WORLD. Why do we require one communicator?</span><o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>3.
It appears that failed slaves cannot be simply respawned. Is this what a repair
would do anyway?</span><o:p></o:p></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>Best
regards.</span><o:p></o:p></p>
<p class=MsoNormal> <o:p></o:p></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Arial","sans-serif"'>Alexander</span><o:p></o:p></p>
<p class=MsoNormal><o:p> </o:p></p>
<div class=MsoNormal align=center style='text-align:center'>
<hr size=2 width="100%" align=center>
</div>
<p class=MsoNormal style='margin-bottom:12.0pt'><b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'>From:</span></b><span style='font-size:10.0pt;
font-family:"Tahoma","sans-serif"'> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Erez Haba<br>
<b>Sent:</b> Wednesday, February 18, 2009 3:53 AM<br>
<b>To:</b> MPI 3.0 Fault Tolerance and Dynamic Process Control working Group<br>
<b>Subject:</b> [Mpi3-ft] MPI Fault Tolerance scenarios</span><o:p></o:p></p>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Hello
all,<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>In
our last meeting we decided to build a set of FT scenarios/programs to help us
understand the details of the interface need to support those scenarios. We
also decided to start with very simple scenarios and add more complex ones as
we understand the former better. I hope that starting with simple
scenarios will help us build a solid foundation on which we can build the more
complex solutions.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>When
we build an FT solution we will focus on the scenario as described, without
complicating the solution just because it would be needed later for a more
complex one. The time will come later to modify the solution as we acquire more
knowledge and built the foundations. Hence, any proposal or change that we make
needs to fit <i><u>exactly</u></i> the scenario (and all those that we
previously looked at) but no more.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>For
example in the first scenario that we’ll look at there is no need for
saving communicator state or error callback; but they might be required later.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Note
that these scenarios focus on process FT rather than checkpoint/restart or
network degradation. I assume we’ll do the latter later.<o:p></o:p></span></p>
</div>
<div style='margin-top:24.0pt'>
<p class=MsoNormal><b><span style='font-size:13.5pt;font-family:"Cambria","serif";
color:#365F91'>Scenario #1: Very Simple Master-Workers</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Description</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
is a very simple master-workers scenario. However simple, we were asked many
times by customers to support FT in this scenario.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>In
this case the MPI application running with n processes, where rank 0 is used as
the master and n-1 ranks are used as workers. The master generates work
(either by getting it directly from user input, or reading a file) and sends it
for processing to a free worker rank. The master sends requests and receives
replies using MPI point-to-point communication. The workers wait for the
incoming message, upon arrival the worker computes the result and sends it back
to the master. The master stores the result to a log file.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Hardening</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>: The goal is to
harden the workers, the master itself is not FT, thus if it fails the entire
application fails. In this case the workers are FT, and are replaced to keep
computation power for this application. (a twist: if a worker cannot be
recovered the master can work with a smaller set of clients up to a low
watermark).<o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Worker</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
worker waits on a blocking receive when a message arrives it process it. If a <i>done</i>
message arrives the worker finalizes MPI and exit normally.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Hardening</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>: There is not
special requirement for hardening here. If the worker encounters a
communication problem with the master, it means that the master is down and it’s
okay to abort the entire job. Thus, it will use the default error handler
(which aborts on errors). Note that we do not need to modify the client
at all to make the application FT (except the master).<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Pseudo
code for the hardened worker:<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:#548DD4;background:white'>int</span><span style='font-size:10.0pt;
font-family:"Courier New";color:black;background:white'> main()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#984806'>MPI_Init</span>()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#548DD4'>for</span>(;;)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#984806'>MPI_Recv</span>(src=0, &query, MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#548DD4'>if</span>(is_done_msg(query))</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
<b>break</b>;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
process_query(&query, &answer);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#984806'>MPI_Send</span>(dst=0, &answer, MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> }</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#984806'>MPI_Finalize</span>()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Notice
that for this FT code there is no requirement for the worker to rejoin the
comm. As the only communicator used is MPI_COMM_WORLD.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Master</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master code reads queries from a stream and passes them on to the workers to
process. The master goes through several phases. In the initialization phase it
sends the first request to each one of the ranks; in the second one it shuts
down any unnecessary ranks (if the job is too small); I the third phase it
enters its progress engine where it handles replies (answers), process recovery
and termination (on input end).<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Hardening</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>: It is the
responsibility of the master to restart any failing workers and make sure that
the request (query) did not get lost if a worker fails. Hence, every time an
error is detected the master will move the worker into repairing state and move
its workload to other workers.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master runs with errors returned rather than aborted<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>One
thing to note about the following code: it is not optimized. I did not try to overlap
computation with communication (which is possible) I tried to keep it as simple
as possible for the purpose of discussion.</span></i><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Pseudo
code for the hardened master; the code needed for repairing the failed ranks is
highlighted in yellow.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:#548DD4;background:white'>int</span><span style='font-size:10.0pt;
font-family:"Courier New";color:black;background:white'> main()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#984806'>MPI_Init</span>()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span style='color:#984806'>MPI_Comm_set_errhandler</span>(MPI_COMM_WORLD,
MPI_ERRORS_RETURN);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#984806'>MPI_Comm_size</span>(MPI_COMM_WORLD,
&n);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> MPI_Request r[n] = MPI_REQUEST_NULL;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> QueryMessage q[n];</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> AnswerMessage a[n];</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#548DD4'>int</span>
active_workers = 0;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span style='color:#548DD4'>bool</span> repairing[n]
= false;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> // Phase 1: send initial requests</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#548DD4'>for</span>(int
i = 1; i < n; i++)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#548DD4'>if</span>(get_next_query(stream, &q[i]) == eof)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
<span style='color:#548DD4'>break</span>;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> active_workers++;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#984806'>MPI_Send</span>(dest=i, &q[i], MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> rc = <span
style='color:#984806'>MPI_Irecv</span>(src=i, buffer=&a[x],
request=&r[x], MPI_COMM_WORLD)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span style='color:
#548DD4'>if</span>(rc != MPI_SUCCESS)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
start_repair(i, repairing, q, a, r, stream); </span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> }</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> }</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> // Phase 2: finalize any unnecessary ranks</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#548DD4'>for</span>(int
i = active_workers + 1; i < n; i++)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#984806'>MPI_Send</span>(dest=i, &done_msg, MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> }</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> // The progress engine. Get answers; send
new requests and handle</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> // process repairs</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#548DD4'>while</span>(active_workers
!= 0)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> rc = <span
style='color:#984806'>MPI_Waitany</span>(n, r, &i, MPI_STATUS_IGNORE);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span style='color:
#548DD4'>if</span>(rc != MPI_SUCCESS)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span
style='color:#548DD4'>if</span>(!repairing[i])</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
start_repair(i, repairing, q, a, r, stream)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> }</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
<span style='color:#548DD4'>else</span></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
active_workers--;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
continue;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> }</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> repairing[i] = false;<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
process_answer(&a[i]);</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#548DD4'>if</span>(get_next_input(stream, &q[i]) == eof)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
active_workers--;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
<span style='color:#984806'>MPI_Send</span>(dest=i, &done_msg)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span
style='color:#548DD4'>else</span></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
<span style='color:#984806'>MPI_Send</span>(dest=i, &q[i])</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>
rc = <span style='color:#984806'>MPI_Irecv</span>(src=i, buffer=&a[i],
request=&r[i], MPI_COMM_WORLD)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span
style='color:#548DD4'>if</span>(rc != MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> {</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>>
start_repair(i, repairing, q, a, r, stream);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> }</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> }</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> }</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'> <span style='color:#984806'>MPI_Finalize</span>()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:white'>}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> <span style='color:#548DD4'>void</span>
start_repair(int i, int repairing[], Query q[], Answer q[], MPI_Request r[],
Stream stream)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> {</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> repairing[i] = true;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> push_query_back(stream,
&q[i]);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> MPI_Comm_Irepair(MPI_COMM_WORLD,
i, &r[i]);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>>>> }</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Cambria","serif";
color:#4F81BD'>Logic description (without FT)</span></b><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master code keeps track of the number of active workers through the
active_workers variable. It is solely used for the purpose of shutdown. When
the master is out of input, it shuts-down the workers by sending them <i>‘done’</i>
message. It decrease the number of active workers and finalizes when this
number reaches zero.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master’s progress engine waits on a vector of requests (note that entry 0
is not used, as to simplify the code); one it gets an answer it processes it
and sends the next query to that worker until it’s out of input.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Cambria","serif";
color:#4F81BD'>Logic description (with FT)</span></b><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master detects a faulty client either synchronously when it ties to initiate an
async receive (no need to check the send, the assumption is that if send
failed, so will the receive call), or async when the async receive completes
with an error. Once an error detected (and identified as a faulty client, more
about this later), the master starts an async repair of that client. If the
repair succeeds, new work is sent to that client. If it does not, the number of
active workers is decreased and the master has to live with less processing
power.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
code above assumes that if the returned code is an error, it should repair the
worker; however as we discussed, there could very well be many different
reasons for an error here, which not all are related to process failure; for
that we might use something in lines of<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal style='text-indent:.5in;background:#D9D9D9'><span
style='font-size:10.0pt;font-family:"Courier New";color:#0070C0;background:
white'>if</span><span style='font-size:10.0pt;font-family:"Courier New";
color:black;background:white'>(</span><span style='font-size:10.0pt;font-family:
"Courier New";color:#984806;background:white'>MPI_Error_event</span><span
style='font-size:10.0pt;font-family:"Courier New";color:black;background:white'>(rc)
== MPI_EVENT_PROCESS_DOWN)...</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>it
would be the responsibility of the MPI implementation to encode or store the
event related to the returned error code.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><i><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>(Note:
in MPICH2 there is a mechanism that enables encoding extended error information
in the error code, which then can be retrieved using MPI_Error_string)</span></i><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Conclusions</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>I
believe that the solution above describes what we have discussed in the last
meeting. The required API’s to support this FT are really minimal but
already cover a good set of users.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Please,
send your comments.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Thoughts?<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Thanks,<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>.Erez<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>P.S.
I will post this on the FT wiki pages (with the feedbac).<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>P.P.S.
there is one more scenario that we discussed, and extension of the
master-workers model. I will try to get it write us as-soon-as-posible.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'> <o:p></o:p></span></p>
</div>
<pre>---------------------------------------------------------------------<o:p></o:p></pre><pre>Intel GmbH<o:p></o:p></pre><pre>Dornacher Strasse 1<o:p></o:p></pre><pre>85622 Feldkirchen/Muenchen Germany<o:p></o:p></pre><pre>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></pre><pre>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></pre><pre>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></pre><pre>VAT Registration No.: DE129385895<o:p></o:p></pre><pre>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></pre><pre><o:p> </o:p></pre><pre>This e-mail and any attachments may contain confidential material for<o:p></o:p></pre><pre>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></pre><pre>by others is strictly prohibited. If you are not the intended<o:p></o:p></pre><pre>recipient, please contact the sender and delete all copies.<o:p></o:p></pre><pre>---------------------------------------------------------------------<o:p></o:p></pre><pre>Intel GmbH<o:p></o:p></pre><pre>Dornacher Strasse 1<o:p></o:p></pre><pre>85622 Feldkirchen/Muenchen Germany<o:p></o:p></pre><pre>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></pre><pre>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></pre><pre>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></pre><pre>VAT Registration No.: DE129385895<o:p></o:p></pre><pre>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></pre><pre><o:p> </o:p></pre><pre>This e-mail and any attachments may contain confidential material for<o:p></o:p></pre><pre>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></pre><pre>by others is strictly prohibited. If you are not the intended<o:p></o:p></pre><pre>recipient, please contact the sender and delete all copies.<o:p></o:p></pre><pre>---------------------------------------------------------------------<o:p></o:p></pre><pre>Intel GmbH<o:p></o:p></pre><pre>Dornacher Strasse 1<o:p></o:p></pre><pre>85622 Feldkirchen/Muenchen Germany<o:p></o:p></pre><pre>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></pre><pre>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></pre><pre>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></pre><pre>VAT Registration No.: DE129385895<o:p></o:p></pre><pre>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></pre><pre><o:p> </o:p></pre><pre>This e-mail and any attachments may contain confidential material for<o:p></o:p></pre><pre>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></pre><pre>by others is strictly prohibited. If you are not the intended<o:p></o:p></pre><pre>recipient, please contact the sender and delete all copies.<o:p></o:p></pre></div>
</body>
</html>