<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<HTML xmlns="http://www.w3.org/TR/REC-html40" xmlns:v =
"urn:schemas-microsoft-com:vml" xmlns:o =
"urn:schemas-microsoft-com:office:office" xmlns:w =
"urn:schemas-microsoft-com:office:word" xmlns:x =
"urn:schemas-microsoft-com:office:excel" xmlns:p =
"urn:schemas-microsoft-com:office:powerpoint" xmlns:a =
"urn:schemas-microsoft-com:office:access" xmlns:dt =
"uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:s =
"uuid:BDC6E3F0-6DA3-11d1-A2A3-00AA00C14882" xmlns:rs =
"urn:schemas-microsoft-com:rowset" xmlns:z = "#RowsetSchema" xmlns:b =
"urn:schemas-microsoft-com:office:publisher" xmlns:ss =
"urn:schemas-microsoft-com:office:spreadsheet" xmlns:c =
"urn:schemas-microsoft-com:office:component:spreadsheet" xmlns:odc =
"urn:schemas-microsoft-com:office:odc" xmlns:oa =
"urn:schemas-microsoft-com:office:activation" xmlns:html =
"http://www.w3.org/TR/REC-html40" xmlns:q =
"http://schemas.xmlsoap.org/soap/envelope/" XMLNS:D = "DAV:" xmlns:mt =
"http://schemas.microsoft.com/sharepoint/soap/meetings/" xmlns:x2 =
"http://schemas.microsoft.com/office/excel/2003/xml" xmlns:ois =
"http://schemas.microsoft.com/sharepoint/soap/ois/" xmlns:dir =
"http://schemas.microsoft.com/sharepoint/soap/directory/" xmlns:ds =
"http://www.w3.org/2000/09/xmldsig#" xmlns:dsp =
"http://schemas.microsoft.com/sharepoint/dsp" xmlns:udc =
"http://schemas.microsoft.com/data/udc" xmlns:xsd =
"http://www.w3.org/2001/XMLSchema" xmlns:sub =
"http://schemas.microsoft.com/sharepoint/soap/2002/1/alerts/" xmlns:ec =
"http://www.w3.org/2001/04/xmlenc#" xmlns:sp =
"http://schemas.microsoft.com/sharepoint/" xmlns:sps =
"http://schemas.microsoft.com/sharepoint/soap/" xmlns:xsi =
"http://www.w3.org/2001/XMLSchema-instance" xmlns:udcs =
"http://schemas.microsoft.com/data/udc/soap" xmlns:udcxf =
"http://schemas.microsoft.com/data/udc/xmlfile" xmlns:udcp2p =
"http://schemas.microsoft.com/data/udc/parttopart" xmlns:wf =
"http://schemas.microsoft.com/sharepoint/soap/workflow/" xmlns:dsss =
"http://schemas.microsoft.com/office/2006/digsig-setup" xmlns:dssi =
"http://schemas.microsoft.com/office/2006/digsig" xmlns:mdssi =
"http://schemas.openxmlformats.org/package/2006/digital-signature" xmlns:mver =
"http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:m =
"http://schemas.microsoft.com/office/2004/12/omml" xmlns:mrels =
"http://schemas.openxmlformats.org/package/2006/relationships" xmlns:spwp =
"http://microsoft.com/sharepoint/webpartpages" xmlns:ex12t =
"http://schemas.microsoft.com/exchange/services/2006/types" xmlns:ex12m =
"http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:pptsl =
"http://schemas.microsoft.com/sharepoint/soap/SlideLibrary/" xmlns:spsl =
"http://microsoft.com/webservices/SharePointPortalServer/PublishedLinksService"
XMLNS:Z = "urn:schemas-microsoft-com:" xmlns:st = ""><HEAD>
<META http-equiv=Content-Type content="text/html; charset=us-ascii">
<META content="MSHTML 6.00.2900.3354" name=GENERATOR><!--[if !mso]>
<STYLE>v\:* {
BEHAVIOR: url(#default#VML)
}
o\:* {
BEHAVIOR: url(#default#VML)
}
w\:* {
BEHAVIOR: url(#default#VML)
}
.shape {
BEHAVIOR: url(#default#VML)
}
</STYLE>
<![endif]-->
<STYLE>@font-face {
font-family: Cambria Math;
}
@font-face {
font-family: Cambria;
}
@font-face {
font-family: Calibri;
}
@font-face {
font-family: Tahoma;
}
@font-face {
font-family: Consolas;
}
@page Section1 {size: 8.5in 11.0in; margin: 1.0in 1.0in 1.0in 1.0in; }
P.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
LI.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
DIV.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
A:link {
COLOR: blue; TEXT-DECORATION: underline; mso-style-priority: 99
}
SPAN.MsoHyperlink {
COLOR: blue; TEXT-DECORATION: underline; mso-style-priority: 99
}
A:visited {
COLOR: purple; TEXT-DECORATION: underline; mso-style-priority: 99
}
SPAN.MsoHyperlinkFollowed {
COLOR: purple; TEXT-DECORATION: underline; mso-style-priority: 99
}
PRE {
FONT-SIZE: 10pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Courier New"; mso-style-priority: 99; mso-style-link: "HTML Preformatted Char"
}
P.emailquote {
BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: medium none; PADDING-LEFT: 0in; FONT-SIZE: 12pt; PADDING-BOTTOM: 0in; MARGIN-LEFT: 1pt; BORDER-LEFT: medium none; MARGIN-RIGHT: 0in; PADDING-TOP: 0in; BORDER-BOTTOM: medium none; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
LI.emailquote {
BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: medium none; PADDING-LEFT: 0in; FONT-SIZE: 12pt; PADDING-BOTTOM: 0in; MARGIN-LEFT: 1pt; BORDER-LEFT: medium none; MARGIN-RIGHT: 0in; PADDING-TOP: 0in; BORDER-BOTTOM: medium none; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
DIV.emailquote {
BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: medium none; PADDING-LEFT: 0in; FONT-SIZE: 12pt; PADDING-BOTTOM: 0in; MARGIN-LEFT: 1pt; BORDER-LEFT: medium none; MARGIN-RIGHT: 0in; PADDING-TOP: 0in; BORDER-BOTTOM: medium none; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
SPAN.HTMLPreformattedChar {
FONT-FAMILY: Consolas; mso-style-priority: 99; mso-style-link: "HTML Preformatted"; mso-style-name: "HTML Preformatted Char"
}
SPAN.EmailStyle20 {
COLOR: #1f497d; FONT-FAMILY: "Calibri","sans-serif"; mso-style-type: personal-reply
}
.MsoChpDefault {
FONT-SIZE: 10pt; mso-style-type: export-only
}
DIV.Section1 {
page: Section1
}
</STYLE>
<!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]--></HEAD>
<BODY lang=EN-US vLink=purple link=blue>
<DIV dir=ltr align=left><SPAN class=790463916-25022009><FONT face=Arial
color=#0000ff size=2>Thanks. What call is taking care of respawning? This is not
crystal clear from the scenario I'm afraid.</FONT></SPAN></DIV><BR>
<DIV class=OutlookMessageHeader lang=en-us dir=ltr align=left>
<HR tabIndex=-1>
<FONT face=Tahoma size=2><B>From:</B> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <B>On Behalf Of </B>Erez
Haba<BR><B>Sent:</B> Wednesday, February 25, 2009 5:35 PM<BR><B>To:</B> MPI 3.0
Fault Tolerance and Dynamic Process Control working Group<BR><B>Subject:</B> Re:
[Mpi3-ft] MPI Fault Tolerance scenarios<BR></FONT><BR></DIV>
<DIV></DIV>
<DIV class=Section1>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Hi
Alexander,<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">As
explained in the intro section, we are building the scenarios bottom up to
create solid foundations for the FT architecture. However you are right and the
scenarios will expand to use collectives and other communicators, just no in
this first scenario.<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">This
scenario is very very simple, and very very basic, to show the absolute minimum
required for process FT. It only uses point-to-point communication over com
world. The workers are stateless and can be respawned without any need for state
recovery.<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Thanks,<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">.Erez<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<DIV>
<DIV
style="BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: #b5c4df 1pt solid; PADDING-LEFT: 0in; PADDING-BOTTOM: 0in; BORDER-LEFT: medium none; PADDING-TOP: 3pt; BORDER-BOTTOM: medium none">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Supalov, Alexander<BR><B>Sent:</B> Wednesday, February 25,
2009 6:25 AM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control
working Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios<o:p></o:p></SPAN></P></DIV></DIV>
<P class=MsoNormal><o:p> </o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Dear
Erez,</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Thank you. A couple
of questions:</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">1. You seem to
restrict communication to pt2pt only. Why? A Bcast upfront could be useful, for
one.</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">2. I can imagine more
complicated communicator combinations than only MPI_COMM_WORLD. Why do we
require one communicator?</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">3. It appears that
failed slaves cannot be simply respawned. Is this what a repair would do
anyway?</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Best
regards.</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Alexander</SPAN><o:p></o:p></P>
<P class=MsoNormal><o:p> </o:p></P>
<DIV class=MsoNormal style="TEXT-ALIGN: center" align=center>
<HR align=center width="100%" SIZE=2>
</DIV>
<P class=MsoNormal style="MARGIN-BOTTOM: 12pt"><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Erez Haba<BR><B>Sent:</B> Wednesday, February 18, 2009 3:53
AM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control working
Group<BR><B>Subject:</B> [Mpi3-ft] MPI Fault Tolerance
scenarios</SPAN><o:p></o:p></P>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hello
all,<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">In our last meeting
we decided to build a set of FT scenarios/programs to help us understand the
details of the interface need to support those scenarios. We also decided to
start with very simple scenarios and add more complex ones as we understand the
former better. I hope that starting with simple scenarios will help us
build a solid foundation on which we can build the more complex
solutions.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">When we build an FT
solution we will focus on the scenario as described, without complicating the
solution just because it would be needed later for a more complex one. The time
will come later to modify the solution as we acquire more knowledge and built
the foundations. Hence, any proposal or change that we make needs to fit
<I><U>exactly</U></I> the scenario (and all those that we previously looked at)
but no more.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">For example in the
first scenario that we’ll look at there is no need for saving communicator state
or error callback; but they might be required later.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Note that these
scenarios focus on process FT rather than checkpoint/restart or network
degradation. I assume we’ll do the latter later.<o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 24pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 13.5pt; COLOR: #365f91; FONT-FAMILY: 'Cambria','serif'">Scenario
#1: Very Simple Master-Workers</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Description</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">This is a very
simple master-workers scenario. However simple, we were asked many times by
customers to support FT in this scenario.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">In this case the
MPI application running with n processes, where rank 0 is used as the master and
n-1 ranks are used as workers. The master generates work (either by
getting it directly from user input, or reading a file) and sends it for
processing to a free worker rank. The master sends requests and receives replies
using MPI point-to-point communication. The workers wait for the incoming
message, upon arrival the worker computes the result and sends it back to the
master. The master stores the result to a log
file.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: The goal is to
harden the workers, the master itself is not FT, thus if it fails the entire
application fails. In this case the workers are FT, and are replaced to keep
computation power for this application. (a twist: if a worker cannot be
recovered the master can work with a smaller set of clients up to a low
watermark).<o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Worker</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The worker waits on
a blocking receive when a message arrives it process it. If a <I>done</I>
message arrives the worker finalizes MPI and exit
normally.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: There is not
special requirement for hardening here. If the worker encounters a communication
problem with the master, it means that the master is down and it’s okay to abort
the entire job. Thus, it will use the default error handler (which aborts on
errors). Note that we do not need to modify the client at all to make the
application FT (except the master).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Pseudo code for the
hardened worker:<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #548dd4; FONT-FAMILY: 'Courier New'">int</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">
main()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Init</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(;;)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Recv</SPAN>(src=0, &query,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(is_done_msg(query))</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<B>break</B>;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
process_query(&query, &answer);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dst=0, &answer,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Finalize</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Notice that for
this FT code there is no requirement for the worker to rejoin the comm. As the
only communicator used is MPI_COMM_WORLD.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Master</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master code
reads queries from a stream and passes them on to the workers to process. The
master goes through several phases. In the initialization phase it sends the
first request to each one of the ranks; in the second one it shuts down any
unnecessary ranks (if the job is too small); I the third phase it enters its
progress engine where it handles replies (answers), process recovery and
termination (on input end).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: It is the
responsibility of the master to restart any failing workers and make sure that
the request (query) did not get lost if a worker fails. Hence, every time an
error is detected the master will move the worker into repairing state and move
its workload to other workers.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master runs
with errors returned rather than aborted<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">One thing to note
about the following code: it is not optimized. I did not try to overlap
computation with communication (which is possible) I tried to keep it as simple
as possible for the purpose of discussion.</SPAN></I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Pseudo code for the
hardened master; the code needed for repairing the failed ranks is highlighted
in yellow.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #548dd4; FONT-FAMILY: 'Courier New'">int</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">
main()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Init</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #984806">MPI_Comm_set_errhandler</SPAN>(MPI_COMM_WORLD,
MPI_ERRORS_RETURN);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Comm_size</SPAN>(MPI_COMM_WORLD,
&n);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
MPI_Request r[n] = MPI_REQUEST_NULL;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
QueryMessage q[n];</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
AnswerMessage a[n];</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">int</SPAN> active_workers = 0;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">bool</SPAN> repairing[n] = false;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// Phase 1: send initial requests</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(int i = 1; i < n; i++)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(get_next_query(stream, &q[i]) ==
eof)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">break</SPAN>;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
active_workers++;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &q[i],
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN style="COLOR: #984806">MPI_Irecv</SPAN>(src=i, buffer=&a[x],
request=&r[x], MPI_COMM_WORLD)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r, stream); </SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// Phase 2: finalize any unnecessary ranks</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(int i = active_workers + 1; i < n;
i++)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &done_msg,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// The progress engine. Get answers; send new requests and handle</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// process repairs</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">while</SPAN>(active_workers != 0)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN style="COLOR: #984806">MPI_Waitany</SPAN>(n, r, &i,
MPI_STATUS_IGNORE);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">if</SPAN>(!repairing[i])</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN
style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r,
stream)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
continue;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
process_answer(&a[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">else</SPAN> <SPAN
style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
active_workers--;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(get_next_input(stream, &q[i]) ==
eof)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
active_workers--;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN
style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &done_msg)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">else</SPAN></SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN
style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &q[i])</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN
style="COLOR: #984806">MPI_Irecv</SPAN>(src=i, buffer=&a[i],
request=&r[i], MPI_COMM_WORLD)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN
style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r,
stream);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Finalize</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">void</SPAN> start_repair(int i, int repairing[],
Query q[], Answer q[], MPI_Request r[], Stream stream)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
repairing[i] = true;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
push_query_back(stream, &q[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
MPI_Comm_Irepair(MPI_COMM_WORLD, i, &r[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Logic
description (without FT)</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master code
keeps track of the number of active workers through the active_workers variable.
It is solely used for the purpose of shutdown. When the master is out of input,
it shuts-down the workers by sending them <I>‘done’</I> message. It decrease the
number of active workers and finalizes when this number reaches
zero.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master’s
progress engine waits on a vector of requests (note that entry 0 is not used, as
to simplify the code); one it gets an answer it processes it and sends the next
query to that worker until it’s out of input.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Logic
description (with FT)</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master detects
a faulty client either synchronously when it ties to initiate an async receive
(no need to check the send, the assumption is that if send failed, so will the
receive call), or async when the async receive completes with an error. Once an
error detected (and identified as a faulty client, more about this later), the
master starts an async repair of that client. If the repair succeeds, new work
is sent to that client. If it does not, the number of active workers is
decreased and the master has to live with less processing
power.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The code above
assumes that if the returned code is an error, it should repair the worker;
however as we discussed, there could very well be many different reasons for an
error here, which not all are related to process failure; for that we might use
something in lines of<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal style="BACKGROUND: #d9d9d9; TEXT-INDENT: 0.5in"><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #0070c0; FONT-FAMILY: 'Courier New'">if</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">(</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #984806; FONT-FAMILY: 'Courier New'">MPI_Error_event</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">(rc)
== MPI_EVENT_PROCESS_DOWN)...</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">it would be the
responsibility of the MPI implementation to encode or store the event related to
the returned error code.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">(Note: in MPICH2
there is a mechanism that enables encoding extended error information in the
error code, which then can be retrieved using MPI_Error_string)</SPAN></I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Conclusions</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">I believe that the
solution above describes what we have discussed in the last meeting. The
required API’s to support this FT are really minimal but already cover a good
set of users.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Please, send your
comments.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Thoughts?<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Thanks,<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">.Erez<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">P.S. I will post
this on the FT wiki pages (with the feedbac).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">P.P.S. there is one
more scenario that we discussed, and extension of the master-workers model. I
will try to get it write us as-soon-as-posible.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV><PRE>---------------------------------------------------------------------<o:p></o:p></PRE><PRE>Intel GmbH<o:p></o:p></PRE><PRE>Dornacher Strasse 1<o:p></o:p></PRE><PRE>85622 Feldkirchen/Muenchen Germany<o:p></o:p></PRE><PRE>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></PRE><PRE>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></PRE><PRE>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></PRE><PRE>VAT Registration No.: DE129385895<o:p></o:p></PRE><PRE>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></PRE><PRE><o:p> </o:p></PRE><PRE>This e-mail and any attachments may contain confidential material for<o:p></o:p></PRE><PRE>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></PRE><PRE>by others is strictly prohibited. If you are not the intended<o:p></o:p></PRE><PRE>recipient, please contact the sender and delete all copies.<o:p></o:p></PRE></DIV><pre>---------------------------------------------------------------------
Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen Germany
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer
Registergericht: Muenchen HRB 47456 Ust.-IdNr.
VAT Registration No.: DE129385895
Citibank Frankfurt (BLZ 502 109 00) 600119052
This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.
</pre></BODY></HTML>