<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:p="urn:schemas-microsoft-com:office:powerpoint" xmlns:a="urn:schemas-microsoft-com:office:access" xmlns:dt="uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:s="uuid:BDC6E3F0-6DA3-11d1-A2A3-00AA00C14882" xmlns:rs="urn:schemas-microsoft-com:rowset" xmlns:z="#RowsetSchema" xmlns:b="urn:schemas-microsoft-com:office:publisher" xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet" xmlns:c="urn:schemas-microsoft-com:office:component:spreadsheet" xmlns:odc="urn:schemas-microsoft-com:office:odc" xmlns:oa="urn:schemas-microsoft-com:office:activation" xmlns:html="http://www.w3.org/TR/REC-html40" xmlns:q="http://schemas.xmlsoap.org/soap/envelope/" xmlns:D="DAV:" xmlns:mt="http://schemas.microsoft.com/sharepoint/soap/meetings/" xmlns:x2="http://schemas.microsoft.com/office/excel/2003/xml" xmlns:ois="http://schemas.microsoft.com/sharepoint/soap/ois/" xmlns:dir="http://schemas.microsoft.com/sharepoint/soap/directory/" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:dsp="http://schemas.microsoft.com/sharepoint/dsp" xmlns:udc="http://schemas.microsoft.com/data/udc" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:sub="http://schemas.microsoft.com/sharepoint/soap/2002/1/alerts/" xmlns:ec="http://www.w3.org/2001/04/xmlenc#" xmlns:sp="http://schemas.microsoft.com/sharepoint/" xmlns:sps="http://schemas.microsoft.com/sharepoint/soap/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:udcs="http://schemas.microsoft.com/data/udc/soap" xmlns:udcxf="http://schemas.microsoft.com/data/udc/xmlfile" xmlns:udcp2p="http://schemas.microsoft.com/data/udc/parttopart" xmlns:wf="http://schemas.microsoft.com/sharepoint/soap/workflow/" xmlns:dsss="http://schemas.microsoft.com/office/2006/digsig-setup" xmlns:dssi="http://schemas.microsoft.com/office/2006/digsig" xmlns:mdssi="http://schemas.openxmlformats.org/package/2006/digital-signature" xmlns:mver="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns:mrels="http://schemas.openxmlformats.org/package/2006/relationships" xmlns:spwp="http://microsoft.com/sharepoint/webpartpages" xmlns:ex12t="http://schemas.microsoft.com/exchange/services/2006/types" xmlns:ex12m="http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:pptsl="http://schemas.microsoft.com/sharepoint/soap/SlideLibrary/" xmlns:spsl="http://microsoft.com/webservices/SharePointPortalServer/PublishedLinksService" xmlns:Z="urn:schemas-microsoft-com:" xmlns:st="" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv=Content-Type content="text/html; charset=us-ascii">
<meta name=Generator content="Microsoft Word 12 (filtered medium)">
<style>
<!--
/* Font Definitions */
@font-face
{font-family:"MS Mincho";
panose-1:2 2 6 9 4 2 5 8 3 4;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Cambria;
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Tahoma;
panose-1:2 11 6 4 3 5 4 4 2 4;}
@font-face
{font-family:"\@MS Mincho";
panose-1:2 2 6 9 4 2 5 8 3 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
p.emailquote, li.emailquote, div.emailquote
{mso-style-name:emailquote;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:1.0pt;
border:none;
padding:0in;
font-size:12.0pt;
font-family:"Times New Roman","serif";}
span.EmailStyle18
{mso-style-type:personal-reply;
font-family:"Calibri","sans-serif";
color:#1F497D;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page Section1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.Section1
{page:Section1;}
-->
</style>
<!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang=EN-US link=blue vlink=purple>
<div class=Section1>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>I’ve uploaded this scenario to the wiki pages; still more wording
needed to complete this scenario.<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>See <a
href="https://svn.mpi-forum.org/trac/mpi-forum-web/wiki/ft/scenarios_and_solutions">https://svn.mpi-forum.org/trac/mpi-forum-web/wiki/ft/scenarios_and_solutions</a><o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>Thanks,<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'>.Erez<o:p></o:p></span></p>
<p class=MsoNormal><span style='font-size:11.0pt;font-family:"Calibri","sans-serif";
color:#1F497D'><o:p> </o:p></span></p>
<div>
<div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>From:</span></b><span
style='font-size:10.0pt;font-family:"Tahoma","sans-serif"'>
mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <b>On Behalf Of </b>Erez Haba<br>
<b>Sent:</b> Tuesday, April 21, 2009 5:59 PM<br>
<b>To:</b> MPI 3.0 Fault Tolerance working Group<br>
<b>Subject:</b> [Mpi3-ft] MPI FT Scenario #2: Master-Leader-Workers<o:p></o:p></span></p>
</div>
</div>
<p class=MsoNormal><o:p> </o:p></p>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Hi
all, here’s the draft code for scenario 2, please take a look.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>(I
still need to beef up the text, but the entire code and concepts are there)<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:24.0pt'>
<p class=MsoNormal><b><span style='font-size:13.5pt;font-family:"Cambria","serif";
color:#365F91'>Scenario 2: Master-Leader-Workers</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>New
MPI API’s in this scenario</span></b><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New"'>MPI_Is_restored_rank(int*
restored);</span></b><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Indicates
if this rank has been restored or is the first start of this process. Can be
defined as a generation counter instead of a flag, however for this scenario
only the flag version was needed.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New"'>MPI_Comm_rejoin(const
char* name, MPI_Comm* newcomm);</span></b><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
interface recreates the communicator context from a named saved context. It
does not restore error handles or any of the attributes (possibly including the
comm name). I does restore all the necessary context for the MPI library to
continue and work with this communicator such as, contextid, comm rank to world
rank mapping and other information that is impl dependent.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New"'>MPI_Comm_save(MPI_Comm
comm, const char* name);</span></b><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
interface saves the communicator context under specific name. it is a
collective operation and all ranks should call this interface with the same
name and communicator. The reason to make this a collective operation is
logical and practical. However from a theoretical pov it does not have to be a
collective call. The reasons are, (1) all ranks have to save the communicator
context to be able to restore if one fails, saving by only some ranks in the
comm has no meaning. (2) a lot of the data is shared between the ranks (like
the rank mapping) saving a single object scales much better than saving an
object per rank. (3) different objects would possibly require different name
for each rank which seems like a complication of the programming model.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Description</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Cambria","serif";
color:#4F81BD'>Helper code</span></b><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>This
code is used but the master, leader and worker to establish the group
communicator. The master has to participate as it is part of the initial split
of comm world.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>const</span></b><span style='font-size:10.0pt;font-family:"Courier New";
color:black'> <b>int</b> x_group_size = 10;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>MPI_Comm</span></b><span style='font-size:10.0pt;font-family:
"Courier New";color:black'> get_workers_group_comm()</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>MPI_Comm</span></b> comm;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>int</span></b> restored = 0;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Is_restored_rank(&restored); </span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(restored)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Get an already saved
communicator context</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // N.B. need to define
the failure semantic.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_rejoin("workers-group-comm", &comm);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>else</span></b></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_rank(MPI_COMM_WORLD, &rank);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
int color = (rank - 1) / x_group_size;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
int key = (rank - 1) % x_group_size;</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_split(MPI_COMM_WORLD, color, key, &comm);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Save the communicator
context to be able to restore it later.</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // note that
communicator attributes and error handler are not saved.</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_save(comm, "workers-group-comm");</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // N.B. The barrier here
guarantees that the comm was saved for all ranks and</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // that recovery is
posible. </span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Barrier(MPI_COMM_WORLD);</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Worker</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
worker code is very similar to the scenario 1 worker code, except that its
using the group communicator.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>// Assumption: fatal errors on comm x would abort only the group of
processes that are part of comm x</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>void</span></b><span style='font-size:10.0pt;font-family:"Courier New";
color:black'> worker()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Init(0, 0);</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm comm = get_workers_group_comm();</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(;;)</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Recv(src=0, &query, comm);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(is_done_message(query))</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>break</span></b>;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
process_query(&query, &answer);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dst=0, &answer, comm);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Finalize()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Leader</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
leader code is the glue between the master and the workers, it redistribute the
worker and reassemble the answers to send back to the master.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>It
is responsible for restarting the master and the workers in case of a failure.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>void</span></b><span style='font-size:10.0pt;font-family:"Courier New";
color:black'> leader()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Init(0, 0)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm comm = get_workers_group_comm();</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
int my_group_size;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_size(comm, &my_group_size);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Set the error handler on comm world to
'errors return' to be able to handle the</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // errors and restart the master or workers if
they fail. Set the error handler</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // *after* creating the group communicator, to
avoid inheriting the error handler.</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>MPI_Comm_set_errhandler(MPI_COMM_WORLD,
MPI_ERRORS_RETURN);</span></span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // process requests coming from the master,
until the *done* message arrives</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(;;)</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
QueryMessage query;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Recv(src=0, &query, MPI_COMM_WORLD);</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc != MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //
Communication with the master failed, try to restart it</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>rc = MPI_Comm_Restart_rank(MPI_COMM_WORLD, 0);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>if</span></b>(rc == MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>continue</span></b>;</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //
Could not restart the master, maybe other leaders could; abort this group</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// and expect the master to restart this leader.</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>MPI_Abort(comm, 1);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(<b><span style='color:#0070C0'>int</span></b>
i = 1; i < my_group_size; i++)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>WorkerQuery</span></b> worker_query;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
compose_worker_query(query, i, &worker_query);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(;;)</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Send(dst=i, worker_query, comm);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc == MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>break</span></b>;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// Cannot communicate with this worker; try to restart it; if successful</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// send the work item again.</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>rc = MPI_Comm_Restart_rank(comm, i);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>if</span></b>(rc == MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>continue</span></b>;</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// Could not start the worker; abort this group and expect the master to</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// restart it.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>MPI_Abort(comm, 2);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(is_done_message(query))</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>break</span></b>;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>AnswerMessage</span></b> answer;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(<b><span style='color:#0070C0'>int</span></b>
i = 1; i < my_group_size; i++)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>WorkerAnswer</span></b> worker_anser;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Recv(src=i, worker_answer, comm);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc !- MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// Cannot receive the answer from this worker, and we can't really retry</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
// abort this group and expect the master to restart it.</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'>
//</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>MPI_Abort(comm, 3);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
agregate_worker_result(worker_answer, i, &answer);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Send the result to
the master; no need to check for error here, we will detect</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // it when with the next
receive.</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dst=0, &answer, MPI_COMM_WORLD);</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Finalize()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div style='margin-top:10.0pt'>
<p class=MsoNormal><b><span style='font-family:"Cambria","serif";color:#4F81BD'>Master</span></b><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>The
master code is very similar to scenario 1. It’s the master responsibility
to restart the leaders in case of a failure.<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>int</span></b><span style='font-size:10.0pt;font-family:"Courier New";
color:black'> leaderof(int group)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>return</span></b> (group - 1) * x_group_size + 1;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><b><span style='font-size:10.0pt;font-family:"Courier New";
color:#0070C0'>void</span></b><span style='font-size:10.0pt;font-family:"Courier New";
color:black'> master()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Init()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // The master does not need a workers group
comm, but in needs to participate in the</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // creation of that comm;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
get_workers_group_comm()</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>MPI_Comm_set_errhandler(MPI_COMM_WORLD,
MPI_ERRORS_RETURN);</span></span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_size(MPI_COMM_WORLD, &world_size);</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // include the master as group 0</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>int</span></b> ngroups = (world_size - 1) /
x_group_size + 1 + 1;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>MPI_Request</span></b> r[ngroups] =
MPI_REQUEST_NULL;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>QueryMessage</span></b> q[ngroups];</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>AnswerMessage</span></b> a[ngroups];</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0'>int</span></b> active_groups = 0;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:#0070C0;background:yellow'>bool</span></b><span
style='background:yellow'> restarting[ngroups] = <b><span style='color:red'>false</span></b>;</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Phase 1: send initial requests</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(<b><span style='color:#0070C0'>int</span></b>
i = 1; i < ngroups; i++)</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(get_next_query(stream, &q[i]) ==
eof)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>break</span></b>;</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
active_groups++;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dest=leaderof(i);, &q[i], MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Irecv(src=leaderof(i), buffer=&a[i], request=&r[i],
MPI_COMM_WORLD)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc != MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>restart_leader(i, restarting, q, a, r, stream);</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // Phase 2: finalize any unnecessary ranks</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>for</span></b>(<b><span style='color:#0070C0'>int</span></b>
i = active_groups + 1; i < ngroups; i++)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dest=leaderof(i), &done_msg, MPI_COMM_WORLD);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // The progress engine. Get answers; send new
requests and handle</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> // process restarts</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
color:gray'> //</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
while(active_groups != 0)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Waitany(ngroups, r, &i, MPI_STATUS_IGNORE);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc != MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red;background:yellow'>if</span></b><span
style='background:yellow'>(!restarting[i])</span></span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
restart_leader(i, restarting, q, a, r, stream)</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>else</span></b></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
active_groups--;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New";
background:yellow'>
<b><span style='color:red'>continue</span></b>;</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<span style='background:yellow'>restarting[i] = false;</span></span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
process_answer(&a[i]);</span><span style='font-size:10.0pt;font-family:
"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(get_next_input(stream, &q[i]) ==
eof)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
active_groups--;</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dest=leaderof(i), &done_msg)</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>else</span></b></span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Send(dest=leaderof(i), &q[i])</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
rc = MPI_Irecv(src=leaderof(i), buffer=&a[i], request=&r[i],
MPI_COMM_WORLD)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
<b><span style='color:red'>if</span></b>(rc != MPI_SUCCESS)</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
{</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
restart_leader(i, restarting, q, a, r, stream);</span><span style='font-size:
10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
}</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Finalize();</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>void
restart_leader(int i, int restarting[], Query q[], Answer q[], MPI_Request r[],
Stream stream)</span><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>{</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
restarting[i] = <b><span style='color:red'>true</span></b>;</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
push_query_back(stream, &q[i]);</span><span style='font-size:10.0pt;
font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>
MPI_Comm_Irestart_rank(MPI_COMM_WORLD, leaderof(i), &r[i]);</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Courier New"'>}</span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>Thanks,<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt;font-family:"Calibri","sans-serif"'>.Erez<o:p></o:p></span></p>
</div>
<div>
<p class=MsoNormal><span style='font-size:10.0pt'> </span><span
style='font-size:10.0pt;font-family:"Calibri","sans-serif"'><o:p></o:p></span></p>
</div>
</div>
</body>
</html>