<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<HTML xmlns="http://www.w3.org/TR/REC-html40" xmlns:v =
"urn:schemas-microsoft-com:vml" xmlns:o =
"urn:schemas-microsoft-com:office:office" xmlns:w =
"urn:schemas-microsoft-com:office:word" xmlns:x =
"urn:schemas-microsoft-com:office:excel" xmlns:p =
"urn:schemas-microsoft-com:office:powerpoint" xmlns:a =
"urn:schemas-microsoft-com:office:access" xmlns:dt =
"uuid:C2F41010-65B3-11d1-A29F-00AA00C14882" xmlns:s =
"uuid:BDC6E3F0-6DA3-11d1-A2A3-00AA00C14882" xmlns:rs =
"urn:schemas-microsoft-com:rowset" xmlns:z = "#RowsetSchema" xmlns:b =
"urn:schemas-microsoft-com:office:publisher" xmlns:ss =
"urn:schemas-microsoft-com:office:spreadsheet" xmlns:c =
"urn:schemas-microsoft-com:office:component:spreadsheet" xmlns:odc =
"urn:schemas-microsoft-com:office:odc" xmlns:oa =
"urn:schemas-microsoft-com:office:activation" xmlns:html =
"http://www.w3.org/TR/REC-html40" xmlns:q =
"http://schemas.xmlsoap.org/soap/envelope/" XMLNS:D = "DAV:" xmlns:mt =
"http://schemas.microsoft.com/sharepoint/soap/meetings/" xmlns:x2 =
"http://schemas.microsoft.com/office/excel/2003/xml" xmlns:ois =
"http://schemas.microsoft.com/sharepoint/soap/ois/" xmlns:dir =
"http://schemas.microsoft.com/sharepoint/soap/directory/" xmlns:ds =
"http://www.w3.org/2000/09/xmldsig#" xmlns:dsp =
"http://schemas.microsoft.com/sharepoint/dsp" xmlns:udc =
"http://schemas.microsoft.com/data/udc" xmlns:xsd =
"http://www.w3.org/2001/XMLSchema" xmlns:sub =
"http://schemas.microsoft.com/sharepoint/soap/2002/1/alerts/" xmlns:ec =
"http://www.w3.org/2001/04/xmlenc#" xmlns:sp =
"http://schemas.microsoft.com/sharepoint/" xmlns:sps =
"http://schemas.microsoft.com/sharepoint/soap/" xmlns:xsi =
"http://www.w3.org/2001/XMLSchema-instance" xmlns:udcs =
"http://schemas.microsoft.com/data/udc/soap" xmlns:udcxf =
"http://schemas.microsoft.com/data/udc/xmlfile" xmlns:udcp2p =
"http://schemas.microsoft.com/data/udc/parttopart" xmlns:wf =
"http://schemas.microsoft.com/sharepoint/soap/workflow/" xmlns:dsss =
"http://schemas.microsoft.com/office/2006/digsig-setup" xmlns:dssi =
"http://schemas.microsoft.com/office/2006/digsig" xmlns:mdssi =
"http://schemas.openxmlformats.org/package/2006/digital-signature" xmlns:mver =
"http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:m =
"http://schemas.microsoft.com/office/2004/12/omml" xmlns:mrels =
"http://schemas.openxmlformats.org/package/2006/relationships" xmlns:spwp =
"http://microsoft.com/sharepoint/webpartpages" xmlns:ex12t =
"http://schemas.microsoft.com/exchange/services/2006/types" xmlns:ex12m =
"http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:pptsl =
"http://schemas.microsoft.com/sharepoint/soap/SlideLibrary/" xmlns:spsl =
"http://microsoft.com/webservices/SharePointPortalServer/PublishedLinksService"
XMLNS:Z = "urn:schemas-microsoft-com:" xmlns:st = ""><HEAD>
<META http-equiv=Content-Type content="text/html; charset=us-ascii">
<META content="MSHTML 6.00.2900.3354" name=GENERATOR><!--[if !mso]>
<STYLE>v\:* {
BEHAVIOR: url(#default#VML)
}
o\:* {
BEHAVIOR: url(#default#VML)
}
w\:* {
BEHAVIOR: url(#default#VML)
}
.shape {
BEHAVIOR: url(#default#VML)
}
</STYLE>
<![endif]-->
<STYLE>@font-face {
font-family: Wingdings;
}
@font-face {
font-family: Cambria Math;
}
@font-face {
font-family: Cambria;
}
@font-face {
font-family: Calibri;
}
@font-face {
font-family: Tahoma;
}
@font-face {
font-family: Consolas;
}
@page Section1 {size: 8.5in 11.0in; margin: 1.0in 1.0in 1.0in 1.0in; }
P.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
LI.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
DIV.MsoNormal {
FONT-SIZE: 12pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Times New Roman","serif"
}
A:link {
COLOR: blue; TEXT-DECORATION: underline; mso-style-priority: 99
}
SPAN.MsoHyperlink {
COLOR: blue; TEXT-DECORATION: underline; mso-style-priority: 99
}
A:visited {
COLOR: purple; TEXT-DECORATION: underline; mso-style-priority: 99
}
SPAN.MsoHyperlinkFollowed {
COLOR: purple; TEXT-DECORATION: underline; mso-style-priority: 99
}
PRE {
FONT-SIZE: 10pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Courier New"; mso-style-priority: 99; mso-style-link: "HTML Preformatted Char"
}
P.MsoAcetate {
FONT-SIZE: 8pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Tahoma","sans-serif"; mso-style-priority: 99; mso-style-link: "Balloon Text Char"
}
LI.MsoAcetate {
FONT-SIZE: 8pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Tahoma","sans-serif"; mso-style-priority: 99; mso-style-link: "Balloon Text Char"
}
DIV.MsoAcetate {
FONT-SIZE: 8pt; MARGIN: 0in 0in 0pt; FONT-FAMILY: "Tahoma","sans-serif"; mso-style-priority: 99; mso-style-link: "Balloon Text Char"
}
SPAN.HTMLPreformattedChar {
FONT-FAMILY: Consolas; mso-style-priority: 99; mso-style-link: "HTML Preformatted"; mso-style-name: "HTML Preformatted Char"
}
SPAN.BalloonTextChar {
FONT-FAMILY: "Tahoma","sans-serif"; mso-style-priority: 99; mso-style-link: "Balloon Text"; mso-style-name: "Balloon Text Char"
}
P.emailquote {
FONT-SIZE: 12pt; MARGIN-LEFT: 1pt; MARGIN-RIGHT: 0in; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
LI.emailquote {
FONT-SIZE: 12pt; MARGIN-LEFT: 1pt; MARGIN-RIGHT: 0in; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
DIV.emailquote {
FONT-SIZE: 12pt; MARGIN-LEFT: 1pt; MARGIN-RIGHT: 0in; FONT-FAMILY: "Times New Roman","serif"; mso-style-name: emailquote; mso-margin-top-alt: auto; mso-margin-bottom-alt: auto
}
SPAN.EmailStyle22 {
COLOR: #1f497d; FONT-FAMILY: "Calibri","sans-serif"; mso-style-type: personal
}
SPAN.EmailStyle23 {
COLOR: #1f497d; FONT-FAMILY: "Calibri","sans-serif"; mso-style-type: personal
}
SPAN.EmailStyle24 {
COLOR: #1f497d; FONT-FAMILY: "Calibri","sans-serif"; mso-style-type: personal-reply
}
.MsoChpDefault {
FONT-SIZE: 10pt; mso-style-type: export-only
}
DIV.Section1 {
page: Section1
}
</STYLE>
<!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]--></HEAD>
<BODY lang=EN-US vLink=purple link=blue>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Dear Erez,</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Thank you. I reply to your notes below in detail, but I
want to say a couple of words upfront: I'm not trying to say that the
proposed interface cannot cover this or that particular use case. Moreover, I
see substantial value in the attempt to identify a minimum, intrinsically
unambiguous, and elegant set of elementary calls that could be used to express
certain FT concepts.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>What I'm trying to say is that making a programmer care
of whether a call failed or not after every MPI call is certainly a way to make
people wrap all MPI calls into their own FT-tolerant wrappers that will do the
job or analyzing the error codes, calling those or these recovery calls,
etc.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>On this backdrop of expected user behavior, we may just
as well resort to the existing error handling mechanism and make it fix faults
according to a certain policy. Whether or not those error handlers have to use
particular API calls is open.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Note also that by introducing a certain API we restrict
the set of options available to the MPI implementation that wants to implement
certain FT policies. In the proposed approach, it will have to have those API
calls, which may not be natural for the platform involved.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>What I'm proposing is:</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN class=053431212-26022009>-
Define a standard set of FT error handlers for a defined set of policies (see,
e.g., Bronis' proposal aired at the telecon: nothing, local MPI calls, pt2pt w/o
affected process, coll w/o affected process, pt2pt w/ affected process, coll w/
affected process)</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN class=053431212-26022009>-
Provide a prototype implementation of those handlers in terms of the proposed
API</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN class=053431212-26022009>-
Allow implementors to provide error handlers that do not use this API, if they
find a better way of fixing particular FT issues up</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN class=053431212-26022009>-
Allow users to create their own error handlers using the proposed API, to which
end the API should be functional inside an error handler</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>In some sense, the crossroads we're standing on now
resemble the situation with MPI-IO back then: everybody was using parallel file
I/O modes, with some success, and then MPI-IO came along with a pretty
involved, multilevel datatype description. Although flexible in the utmost
sense, this representation effectively complicated the I/O implementor's job, at
least for starters, and now, about 15 years since its inception, we still see
very interesting papers that describe how, basically, implement all those
well known parallel I/O modes efficiently when datatypes look so or so. This is
a good and challenging programming task, sure. Is this what users actually need?
I doubt it. Moreover, I bet that should someone come up and propose a higher
level I/O abstraction that would replace the current I/O interface with
something easy to understand, parallel I/O would suddenly become a commodity
rather than a rarely seen beast it is now, at least in the commercial ISV
space.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Note that I'm not trying to rubbish the venerable
MPI-IO. What I'm proposing is that instead of falling into the same trap,
we should look at MPI-IO and think of the consequences of what we are doing.
Then, while analyzing the possible elementary interface, we should still
keep in mind that 99% of the users won't deal with it if provided with a
simple and easy to comprehend set of standard FT error handlers. And believe me,
they won't care about how they are implemented as long as they work as expected.
And this in turn will increase the chances for MPI FT to be used in the field,
and thus pass the ultimate usability test I mentioned
elsewhere.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN class=053431212-26022009>A
part of the discussion background is that we still has not settled, as a group,
whether MPI-3 is going to be the interface for high-end HPC only, or also for a
wider and wider masses of programmers. Making things complicated favors high end
HPC. Making things easy to use favors mass market. Where is MPI future? Let's
think about that, make the decision, and then implement it. Otherwise we'll be
always oscillating between high end and mass market, and the resulting solution
will most likely please no-one.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Best regards.</SPAN></FONT></DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </DIV>
<DIV dir=ltr align=left><FONT face=Arial size=2><SPAN
class=053431212-26022009>Alexander</SPAN></FONT></DIV><BR>
<DIV class=OutlookMessageHeader lang=en-us dir=ltr align=left>
<HR tabIndex=-1>
<FONT face=Tahoma size=2><B>From:</B> mpi3-ft-bounces@lists.mpi-forum.org
[mailto:mpi3-ft-bounces@lists.mpi-forum.org] <B>On Behalf Of </B>Erez
Haba<BR><B>Sent:</B> Thursday, February 26, 2009 7:21 AM<BR><B>To:</B> MPI 3.0
Fault Tolerance and Dynamic Process Control working Group<BR><B>Subject:</B> Re:
[Mpi3-ft] MPI Fault Tolerance scenarios<BR></FONT><BR></DIV>
<DIV></DIV>
<DIV class=Section1>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Thanks
Alexander, please see inline…<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<DIV>
<DIV
style="BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: #b5c4df 1pt solid; PADDING-LEFT: 0in; PADDING-BOTTOM: 0in; BORDER-LEFT: medium none; PADDING-TOP: 3pt; BORDER-BOTTOM: medium none">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Supalov, Alexander<BR><B>Sent:</B> Wednesday, February 25,
2009 2:20 PM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control
working Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios<o:p></o:p></SPAN></P></DIV></DIV>
<P class=MsoNormal><o:p> </o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">Thanks.
What error handler will be in effect when the MPI_Comm_Irepair is called? If
it's MPI_ERRORS_ARE_FATAL, an error will terminate the application. If it's
MPI_ERRORS_RETURN, one will either have to make MPI_Comm_Irepair and some other
proposed FT calls an exception compared to the rest of the MPI standard, or
explicitly set the error handler on the respective comm to
MPI_ERRORS_RETURN.</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">[erezh]
This server side code sets the error handler to MPI_ERROR_RETURNS. I don’t see
the need for any special case in this API, or am I missing
something?<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">In
generate if your application is FT you should not have MPI_ERRORS_ARE_FATAL as
your error handler as It beats the purpose of FT
app.<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p><SPAN
class=053431212-26022009><FONT face=Arial color=#000000 size=2>OK. I think we
just need to introduce more error handlers in this case, see
below.</FONT></SPAN></o:p></SPAN></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">And if
one anyway has to set the error handler to something else that
MPI_ERRORS_ARE_FATAL, why not delegating this whole comm restoration stuff to
the error handler? The handler could call the MPI_Comm_Irepair or anything else
required, and either return control to the application on success, or abort it
using MPI_Abort on the respective communicator.</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">[erezh]
that is possible to handle to error in your error handler. However this makes
the error handler code synchronous and would block the app until the failed
process is restarted. This might take some time, which is why I used the async
version of repair in the master’s code, hence using Irepair and not
repair.<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Two
more things though, one I don’t want to abort in the case that I could not
respawn the worker. I rather just shrink the number of workers. I don’t think
it’s possible to do that in an error handler. Second, I’d like to reassign the
work to another worker. I can’t do that if the error is detected when the
receive is posted, and anyhow the error handler might be called twice in this
case, once during the send and once during the receive. I think that would have
complicated the code to distinguish if a repair was already
invoked.</SPAN></I></B><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal> </P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN class=053431212-26022009>What
would complicate the code is the necessity to do FT due diligence after each and
every MPI call, which appears to be the order of the day. Also, by the time you
have a fault, you're probably not so much concerned about performance as about
job survival. So, paying a little overhead for the process startup appears to be
a reasonable tradeoff here.</SPAN></FONT></P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN class=053431212-26022009>Also,
note that waiting for the process to be re-created also resolves the problem of
a process being unavailable for a spell. While the error handler is active, and
this can easily be done in a reentrant manner, this process simply won't be
available.</SPAN></FONT></P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN
class=053431212-26022009></SPAN></FONT> </P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN class=053431212-26022009>The
shrinking and reassignment can just as well be done either from inside the error
handler or, if that's too cumbersome, by setting a certain variable there and
allowing the main control loop of the master program reassign the piece of work
as appropriate.</SPAN></FONT></P>
<P class=MsoNormal><FONT face=Arial size=2></FONT><o:p><FONT face=Arial
size=2></FONT></o:p> </P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">By
defining several standard error handlers for various fault tolerance policies we
could implement what Bronis was proposing I guess. Note that these error
handlers could either use the proposed low level API, or do something
implementation specific do achieve their stated goal.</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">[erezh]
I think that it depends very much on the application. For some it might work,
but for others I don’t think it would.<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><SPAN><STRONG><EM>If you take this simple scenario code, I
don’t think it would be simple as you suggest. The application need to be
notified when a repair failed and need to decide to continue and run with less
workers. I think that would be hard to accomplish with pre-canned
solutions.</EM></STRONG><SPAN class=053431212-26022009><FONT face=Arial
size=2> </FONT></SPAN></SPAN></P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2></FONT></SPAN></SPAN> </P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2>See above. Error handler can either do this or notify the
application accordingly.</FONT> </SPAN><o:p></o:p></SPAN></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">Also,
different implementations could have different sets of provided standard error
handlers, and thus vary the level of the fault tolerance support provided,
which is what I'd be looking for. This includes providing no FT support at all
by defining no special FT error handlers, thus making the FT stuff completely
orthogonal to the rest of the standard.</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN><STRONG><EM>[erezh] I think that at the moment we’re
trying to understand the primitives required for FT, rather that jump into
defining everyone favorite programming model. By being explicit we understand
the primitives and constructs needed. Once we done that, we can go and recan the
model into some fancy API/set of fags/…</EM></STRONG><SPAN
class=053431212-26022009><FONT face=Arial size=2> </FONT></SPAN></SPAN></P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2></FONT></SPAN></SPAN> </P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2>I'm concerned that while we're trying to understand primitives in this
particular or another scenario, we may be missing the whole point of FT:
people may not want to deal with this stuff explicitly at every particular place
of the program. They want reliable connections, and a chance to react to
occasional unreliability.</FONT></SPAN></SPAN></P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2></FONT></SPAN></SPAN> </P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2>Whether or not one has to use that or this interface for this is an open
question. Delegating the FT stuff to the error handlers that might use some
internal mechanisms different from those envisioned currently might help to
make the standard more flexible and amenable to future architectural
advances.</FONT> </SPAN><o:p></o:p></SPAN></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">And
finally, this would indeed make FT transparent to the application. The proposed
use of the low level API litters the application code with error </SPAN><SPAN
style="FONT-SIZE: 10pt; COLOR: #1f497d; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">checks
and reaction to every possible error condition in the application code. This
does not seem to be the right way.</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">[erezh]
I think we had the general agreement in the FT workgroup that we can NOT
make FT transparent, nether this is a goal for the
WG.<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">In
any case, shouldn’t you check for returned error code in your app anyway?
Otherwise you run into corrupted data bugs. (or you don’t
care?)<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">I
can think of a nicer programming model where instead of returned errors you get
exception and you have your error handling routines dealing with the errors… oh
wait we have that with the C++ bindings… </SPAN></I></B><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: Wingdings">J</SPAN></I></B><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><FONT face=Arial size=2></FONT> </P>
<P class=MsoNormal><FONT face=Arial size=2><SPAN class=053431212-26022009>For
that to happen, the proposed routines should be functional inside error
handlers. I think this is a strong requirement that should be added to the
current spec.</SPAN></FONT></P>
<P class=MsoNormal><FONT face=Arial size=2></FONT> </P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">The
proposed error handling approach is backward compatible, flexible, elegant, and
simple. Thoughts?</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">[erezh]
do you mean you get FT for free?<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">The
current explicit API’s are backward compatible. They have zero affect on
existing applications. I think that the model we are driving toward is flexible,
elegant and simple… plus preferment. You pay extra only when you care about FT
and even that can be amortized.<o:p></o:p></SPAN></I></B></P>
<P class=MsoNormal><B><I><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></I></B></P>
<P class=MsoNormal><SPAN><STRONG><EM>I think that at the moment you only see a
fraction of the model. With few more scenarios I think that you’ll be able to
see the complexity of the problem, plus the beauty of the solution we’re working
towards.</EM></STRONG><SPAN class=053431212-26022009><FONT face=Arial
size=2> </FONT></SPAN></SPAN></P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2></FONT></SPAN></SPAN> </P>
<P class=MsoNormal><SPAN><SPAN class=053431212-26022009><FONT face=Arial
size=2>Let's see that first.</FONT> </SPAN><o:p></o:p></SPAN></P>
<P class=MsoNormal><o:p><FONT face=Arial size=2></FONT></o:p> </P>
<DIV class=MsoNormal style="TEXT-ALIGN: center" align=center>
<HR align=center width="100%" SIZE=2>
</DIV>
<P class=MsoNormal style="MARGIN-BOTTOM: 12pt"><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Erez Haba<BR><B>Sent:</B> Wednesday, February 25, 2009 5:45
PM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control working
Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">MPI_Comm_Irepair<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<DIV>
<DIV
style="BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: #b5c4df 1pt solid; PADDING-LEFT: 0in; PADDING-BOTTOM: 0in; BORDER-LEFT: medium none; PADDING-TOP: 3pt; BORDER-BOTTOM: medium none">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Supalov, Alexander<BR><B>Sent:</B> Wednesday, February 25,
2009 8:40 AM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control
working Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios<o:p></o:p></SPAN></P></DIV></DIV>
<P class=MsoNormal><o:p> </o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; COLOR: blue; FONT-FAMILY: 'Arial','sans-serif'">Thanks.
What call is taking care of respawning? This is not crystal clear from the
scenario I'm afraid.</SPAN><o:p></o:p></P>
<P class=MsoNormal><o:p> </o:p></P>
<DIV class=MsoNormal style="TEXT-ALIGN: center" align=center>
<HR align=center width="100%" SIZE=2>
</DIV>
<P class=MsoNormal style="MARGIN-BOTTOM: 12pt"><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Erez Haba<BR><B>Sent:</B> Wednesday, February 25, 2009 5:35
PM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control working
Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Hi
Alexander,<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">As
explained in the intro section, we are building the scenarios bottom up to
create solid foundations for the FT architecture. However you are right and the
scenarios will expand to use collectives and other communicators, just no in
this first scenario.<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">This
scenario is very very simple, and very very basic, to show the absolute minimum
required for process FT. It only uses point-to-point communication over com
world. The workers are stateless and can be respawned without any need for state
recovery.<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">Thanks,<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'">.Erez<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 11pt; COLOR: #1f497d; FONT-FAMILY: 'Calibri','sans-serif'"><o:p> </o:p></SPAN></P>
<DIV>
<DIV
style="BORDER-RIGHT: medium none; PADDING-RIGHT: 0in; BORDER-TOP: #b5c4df 1pt solid; PADDING-LEFT: 0in; PADDING-BOTTOM: 0in; BORDER-LEFT: medium none; PADDING-TOP: 3pt; BORDER-BOTTOM: medium none">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Supalov, Alexander<BR><B>Sent:</B> Wednesday, February 25,
2009 6:25 AM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control
working Group<BR><B>Subject:</B> Re: [Mpi3-ft] MPI Fault Tolerance
scenarios<o:p></o:p></SPAN></P></DIV></DIV>
<P class=MsoNormal><o:p> </o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Dear
Erez,</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Thank you. A couple
of questions:</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">1. You seem to
restrict communication to pt2pt only. Why? A Bcast upfront could be useful, for
one.</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">2. I can imagine more
complicated communicator combinations than only MPI_COMM_WORLD. Why do we
require one communicator?</SPAN><o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">3. It appears that
failed slaves cannot be simply respawned. Is this what a repair would do
anyway?</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Best
regards.</SPAN><o:p></o:p></P>
<P class=MsoNormal> <o:p></o:p></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Arial','sans-serif'">Alexander</SPAN><o:p></o:p></P>
<P class=MsoNormal><o:p> </o:p></P>
<DIV class=MsoNormal style="TEXT-ALIGN: center" align=center>
<HR align=center width="100%" SIZE=2>
</DIV>
<P class=MsoNormal style="MARGIN-BOTTOM: 12pt"><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">From:</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Tahoma','sans-serif'">
mpi3-ft-bounces@lists.mpi-forum.org [mailto:mpi3-ft-bounces@lists.mpi-forum.org]
<B>On Behalf Of </B>Erez Haba<BR><B>Sent:</B> Wednesday, February 18, 2009 3:53
AM<BR><B>To:</B> MPI 3.0 Fault Tolerance and Dynamic Process Control working
Group<BR><B>Subject:</B> [Mpi3-ft] MPI Fault Tolerance
scenarios</SPAN><o:p></o:p></P>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hello
all,<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">In our last meeting
we decided to build a set of FT scenarios/programs to help us understand the
details of the interface need to support those scenarios. We also decided to
start with very simple scenarios and add more complex ones as we understand the
former better. I hope that starting with simple scenarios will help us
build a solid foundation on which we can build the more complex
solutions.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">When we build an FT
solution we will focus on the scenario as described, without complicating the
solution just because it would be needed later for a more complex one. The time
will come later to modify the solution as we acquire more knowledge and built
the foundations. Hence, any proposal or change that we make needs to fit
<I><U>exactly</U></I> the scenario (and all those that we previously looked at)
but no more.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">For example in the
first scenario that we’ll look at there is no need for saving communicator state
or error callback; but they might be required later.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Note that these
scenarios focus on process FT rather than checkpoint/restart or network
degradation. I assume we’ll do the latter later.<o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 24pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 13.5pt; COLOR: #365f91; FONT-FAMILY: 'Cambria','serif'">Scenario
#1: Very Simple Master-Workers</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Description</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">This is a very
simple master-workers scenario. However simple, we were asked many times by
customers to support FT in this scenario.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">In this case the
MPI application running with n processes, where rank 0 is used as the master and
n-1 ranks are used as workers. The master generates work (either by
getting it directly from user input, or reading a file) and sends it for
processing to a free worker rank. The master sends requests and receives replies
using MPI point-to-point communication. The workers wait for the incoming
message, upon arrival the worker computes the result and sends it back to the
master. The master stores the result to a log
file.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: The goal is to
harden the workers, the master itself is not FT, thus if it fails the entire
application fails. In this case the workers are FT, and are replaced to keep
computation power for this application. (a twist: if a worker cannot be
recovered the master can work with a smaller set of clients up to a low
watermark).<o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Worker</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The worker waits on
a blocking receive when a message arrives it process it. If a <I>done</I>
message arrives the worker finalizes MPI and exit
normally.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: There is not
special requirement for hardening here. If the worker encounters a communication
problem with the master, it means that the master is down and it’s okay to abort
the entire job. Thus, it will use the default error handler (which aborts on
errors). Note that we do not need to modify the client at all to make the
application FT (except the master).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Pseudo code for the
hardened worker:<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #548dd4; FONT-FAMILY: 'Courier New'">int</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">
main()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Init</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(;;)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Recv</SPAN>(src=0, &query,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(is_done_msg(query))</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<B>break</B>;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
process_query(&query, &answer);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dst=0, &answer,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Finalize</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Notice that for
this FT code there is no requirement for the worker to rejoin the comm. As the
only communicator used is MPI_COMM_WORLD.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Master</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master code
reads queries from a stream and passes them on to the workers to process. The
master goes through several phases. In the initialization phase it sends the
first request to each one of the ranks; in the second one it shuts down any
unnecessary ranks (if the job is too small); I the third phase it enters its
progress engine where it handles replies (answers), process recovery and
termination (on input end).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Hardening</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">: It is the
responsibility of the master to restart any failing workers and make sure that
the request (query) did not get lost if a worker fails. Hence, every time an
error is detected the master will move the worker into repairing state and move
its workload to other workers.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master runs
with errors returned rather than aborted<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">One thing to note
about the following code: it is not optimized. I did not try to overlap
computation with communication (which is possible) I tried to keep it as simple
as possible for the purpose of discussion.</SPAN></I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Pseudo code for the
hardened master; the code needed for repairing the failed ranks is highlighted
in yellow.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #548dd4; FONT-FAMILY: 'Courier New'">int</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">
main()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Init</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #984806">MPI_Comm_set_errhandler</SPAN>(MPI_COMM_WORLD,
MPI_ERRORS_RETURN);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Comm_size</SPAN>(MPI_COMM_WORLD,
&n);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
MPI_Request r[n] = MPI_REQUEST_NULL;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
QueryMessage q[n];</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
AnswerMessage a[n];</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">int</SPAN> active_workers = 0;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">bool</SPAN> repairing[n] = false;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// Phase 1: send initial requests</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(int i = 1; i < n; i++)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(get_next_query(stream, &q[i]) ==
eof)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">break</SPAN>;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
active_workers++;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &q[i],
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN style="COLOR: #984806">MPI_Irecv</SPAN>(src=i, buffer=&a[x],
request=&r[x], MPI_COMM_WORLD)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r, stream); </SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// Phase 2: finalize any unnecessary ranks</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">for</SPAN>(int i = active_workers + 1; i < n;
i++)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &done_msg,
MPI_COMM_WORLD);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// The progress engine. Get answers; send new requests and handle</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
// process repairs</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
//</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">while</SPAN>(active_workers != 0)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN style="COLOR: #984806">MPI_Waitany</SPAN>(n, r, &i,
MPI_STATUS_IGNORE);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN
style="COLOR: #548dd4">if</SPAN>(!repairing[i])</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r,
stream)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">else</SPAN></SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
active_workers--;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
continue;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
repairing[i] = false;<o:p></o:p></SPAN></P>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
process_answer(&a[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">if</SPAN>(get_next_input(stream, &q[i]) ==
eof)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
active_workers--;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN
style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &done_msg)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #548dd4">else</SPAN></SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN
style="COLOR: #984806">MPI_Send</SPAN>(dest=i, &q[i])</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
rc = <SPAN
style="COLOR: #984806">MPI_Irecv</SPAN>(src=i, buffer=&a[i],
request=&r[i], MPI_COMM_WORLD)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN
style="COLOR: #548dd4">if</SPAN>(rc != MPI_SUCCESS)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
start_repair(i, repairing, q, a, r,
stream);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">
<SPAN style="COLOR: #984806">MPI_Finalize</SPAN>()</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; FONT-FAMILY: 'Courier New'">}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
<SPAN style="COLOR: #548dd4">void</SPAN> start_repair(int i, int repairing[],
Query q[], Answer q[], MPI_Request r[], Stream stream)</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
{</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
repairing[i] = true;</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
push_query_back(stream, &q[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
MPI_Comm_Irepair(MPI_COMM_WORLD, i, &r[i]);</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: yellow; FONT-FAMILY: 'Courier New'">>>>
}</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Logic
description (without FT)</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master code
keeps track of the number of active workers through the active_workers variable.
It is solely used for the purpose of shutdown. When the master is out of input,
it shuts-down the workers by sending them <I>‘done’</I> message. It decrease the
number of active workers and finalizes when this number reaches
zero.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master’s
progress engine waits on a vector of requests (note that entry 0 is not used, as
to simplify the code); one it gets an answer it processes it and sends the next
query to that worker until it’s out of input.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="FONT-SIZE: 10pt; COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Logic
description (with FT)</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The master detects
a faulty client either synchronously when it ties to initiate an async receive
(no need to check the send, the assumption is that if send failed, so will the
receive call), or async when the async receive completes with an error. Once an
error detected (and identified as a faulty client, more about this later), the
master starts an async repair of that client. If the repair succeeds, new work
is sent to that client. If it does not, the number of active workers is
decreased and the master has to live with less processing
power.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">The code above
assumes that if the returned code is an error, it should repair the worker;
however as we discussed, there could very well be many different reasons for an
error here, which not all are related to process failure; for that we might use
something in lines of<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal style="BACKGROUND: #d9d9d9; TEXT-INDENT: 0.5in"><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #0070c0; FONT-FAMILY: 'Courier New'">if</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">(</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: #984806; FONT-FAMILY: 'Courier New'">MPI_Error_event</SPAN><SPAN
style="FONT-SIZE: 10pt; BACKGROUND: white; COLOR: black; FONT-FAMILY: 'Courier New'">(rc)
== MPI_EVENT_PROCESS_DOWN)...</SPAN><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">it would be the
responsibility of the MPI implementation to encode or store the event related to
the returned error code.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">(Note: in MPICH2
there is a mechanism that enables encoding extended error information in the
error code, which then can be retrieved using MPI_Error_string)</SPAN></I><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV style="MARGIN-TOP: 10pt">
<P class=MsoNormal><B><SPAN
style="COLOR: #4f81bd; FONT-FAMILY: 'Cambria','serif'">Conclusions</SPAN></B><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"><o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">I believe that the
solution above describes what we have discussed in the last meeting. The
required API’s to support this FT are really minimal but already cover a good
set of users.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Please, send your
comments.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Thoughts?<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">Thanks,<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">.Erez<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">P.S. I will post
this on the FT wiki pages (with the feedbac).<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'">P.P.S. there is one
more scenario that we discussed, and extension of the master-workers model. I
will try to get it write us as-soon-as-posible.<o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV>
<DIV>
<P class=MsoNormal><SPAN
style="FONT-SIZE: 10pt; FONT-FAMILY: 'Calibri','sans-serif'"> <o:p></o:p></SPAN></P></DIV><PRE>---------------------------------------------------------------------<o:p></o:p></PRE><PRE>Intel GmbH<o:p></o:p></PRE><PRE>Dornacher Strasse 1<o:p></o:p></PRE><PRE>85622 Feldkirchen/Muenchen Germany<o:p></o:p></PRE><PRE>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></PRE><PRE>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></PRE><PRE>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></PRE><PRE>VAT Registration No.: DE129385895<o:p></o:p></PRE><PRE>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></PRE><PRE><o:p> </o:p></PRE><PRE>This e-mail and any attachments may contain confidential material for<o:p></o:p></PRE><PRE>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></PRE><PRE>by others is strictly prohibited. If you are not the intended<o:p></o:p></PRE><PRE>recipient, please contact the sender and delete all copies.<o:p></o:p></PRE><PRE>---------------------------------------------------------------------<o:p></o:p></PRE><PRE>Intel GmbH<o:p></o:p></PRE><PRE>Dornacher Strasse 1<o:p></o:p></PRE><PRE>85622 Feldkirchen/Muenchen Germany<o:p></o:p></PRE><PRE>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></PRE><PRE>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></PRE><PRE>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></PRE><PRE>VAT Registration No.: DE129385895<o:p></o:p></PRE><PRE>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></PRE><PRE><o:p> </o:p></PRE><PRE>This e-mail and any attachments may contain confidential material for<o:p></o:p></PRE><PRE>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></PRE><PRE>by others is strictly prohibited. If you are not the intended<o:p></o:p></PRE><PRE>recipient, please contact the sender and delete all copies.<o:p></o:p></PRE><PRE>---------------------------------------------------------------------<o:p></o:p></PRE><PRE>Intel GmbH<o:p></o:p></PRE><PRE>Dornacher Strasse 1<o:p></o:p></PRE><PRE>85622 Feldkirchen/Muenchen Germany<o:p></o:p></PRE><PRE>Sitz der Gesellschaft: Feldkirchen bei Muenchen<o:p></o:p></PRE><PRE>Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer<o:p></o:p></PRE><PRE>Registergericht: Muenchen HRB 47456 Ust.-IdNr.<o:p></o:p></PRE><PRE>VAT Registration No.: DE129385895<o:p></o:p></PRE><PRE>Citibank Frankfurt (BLZ 502 109 00) 600119052<o:p></o:p></PRE><PRE><o:p> </o:p></PRE><PRE>This e-mail and any attachments may contain confidential material for<o:p></o:p></PRE><PRE>the sole use of the intended recipient(s). Any review or distribution<o:p></o:p></PRE><PRE>by others is strictly prohibited. If you are not the intended<o:p></o:p></PRE><PRE>recipient, please contact the sender and delete all copies.<o:p></o:p></PRE></DIV><pre>---------------------------------------------------------------------
Intel GmbH
Dornacher Strasse 1
85622 Feldkirchen/Muenchen Germany
Sitz der Gesellschaft: Feldkirchen bei Muenchen
Geschaeftsfuehrer: Douglas Lusk, Peter Gleissner, Hannes Schwaderer
Registergericht: Muenchen HRB 47456 Ust.-IdNr.
VAT Registration No.: DE129385895
Citibank Frankfurt (BLZ 502 109 00) 600119052
This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.
</pre></BODY></HTML>