<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<!--[if !mso]><style>v\:* {behavior:url(#default#VML);}
o\:* {behavior:url(#default#VML);}
w\:* {behavior:url(#default#VML);}
.shape {behavior:url(#default#VML);}
</style><![endif]--><style><!--
/* Font Definitions */
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:"Segoe UI Emoji";
panose-1:2 11 5 2 4 2 4 2 2 3;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
span.EmailStyle22
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1128819073;
mso-list-type:hybrid;
mso-list-template-ids:-1875353492 -1015526416 134807555 134807557 134807553 134807555 134807557 134807553 134807555 134807557;}
@list l0:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1
{mso-list-id:1697265426;
mso-list-template-ids:-184650644;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:36.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:72.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:108.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:144.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:180.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:216.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:252.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:288.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:324.0pt;
mso-level-number-position:left;
text-indent:-18.0pt;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]-->
</head>
<body lang="EN-GB" link="#0563C1" vlink="#954F72" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Hi Martin,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Responses inline.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Best wishes.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Dan.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt">From:</span></b><span lang="EN-US" style="font-size:11.0pt"> Martin Schulz <schulzm@in.tum.de>
<br>
<b>Sent:</b> 06 January 2023 02:14<br>
<b>To:</b> Holmes, Daniel John <daniel.john.holmes@intel.com>; MPI Sessions working group <mpiwg-sessions@lists.mpi-forum.org><br>
<b>Cc:</b> Pritchard Jr., Howard <howardp@lanl.gov><br>
<b>Subject:</b> Re: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Hi Dan,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">A few comments (kind of many intertwined good discussions
</span><span lang="EN-US" style="font-size:11.0pt;font-family:"Segoe UI Emoji",sans-serif;mso-fareast-language:EN-US">😊</span><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"> ):<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<ul style="margin-top:0cm" type="disc">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l0 level1 lfo3"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Have we ever been able to define the difference between immediate and local? I know there is a perception difference,
but what is difference to the user? What is too long for an immediate call? Are we disallowing progress? If so, what happens if the OS switches threads (this “blocks” the current one) and then the other thread makes progress? I don’t we can define this – I
think for the user these two things will always be the same (from how they have to program) and an implementation will always have the freedom to do what it wants – there could at most be a performance expectation or advice to implementors.<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">[DAN] This has always been the argument against attempting to differentiate between immediate and local; it has always been a successful argument. It means text stating
that persistent point-to-point initialization procedures “involve no communication” (MPI-4.0 section 3.9 page 94 line 37) is an erratum we should fix – how can we say that when the OS might context switch to another thread that does legitimately does communication?
Also, we do define the semantic of immediate without naming it – when we describe the independence of basic runtime routines (MPI-4.0 section 2.9.1, page 28, lines 24-27) – all “library routines that are part of the basic language environment” have this unnamed
immediate semantic, even though the OS could preempt a call to “printf”, context-switch to a different MPI process, and do MPI things.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<ul style="margin-top:0cm" type="disc">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l0 level1 lfo3"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Why should an MPI implementation be prevented from calling the progress engine in a Comm_rank call? Why would
it not be able to communicate, if it wants to? E.g., is a PMPI wrapper implementation that sends an acknowledgement that this call has been made to a central location and waits for it being received an incorrect wrapper?<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">[DAN] Permitting poking the (local) progress engine during the call and requiring some (remote) progress to happen before returning from the call are observably different
– the first cannot delay until remote action, the second can be arbitrarily delayed by refusing to enter MPI at another MPI process. OTOH, the example you give (reliably log procedure call) is not an MPI progress action, so it is irrelevant to the discussion
of immediate vs. local. If the PMPI wrapper used MPI to send the log entry and MPI to ensure it has been received, then the wrapper would be nonlocal, so you cannot mean that. You must be intending usage of some non-MPI communication mechanism, which is a
bad idea in the presence of pending MPI communication (MPI-4.0 section 11.10.5 page 549 line 21 says it’s UB), which makes calling your intercepted MPI_COMM_RANK risky.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<ul style="margin-top:0cm" type="disc">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l0 level1 lfo3"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">As for the Sessions API – I agree with most of what you say on local vs. non-local, but this is the user’s perspective.
I was more asking about whether there is ever a need for an implementation to wait for something to complete. If we can really exclude that, then we should be good – however, I doubt we can do that for sure going forward.<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">[DAN] We can always exclude that wait because we can always postpone that wait.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<ul style="margin-top:0cm" type="disc">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l0 level1 lfo3"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">As for the waiting for changes – shouldn’t this be part of the query of the process list? Why a separate call?<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">[DAN] No. Because we already have the separate calls and all of them have the necessary semantic. The only process sets that could be *<b>discovered</b>* by a subsequent
query call are ones where the calling MPI process is not a member. The existing immediate/local query will check for such changes but not delay until such a change happens. If you want “I know a change is supposed to happen eventually, delay here until it
does and tell me about it” then you need a new probably-nonlocal procedure (it’s return depends on remote action, but not yet an MPI procedure because we have no MPI way to create new process sets). This is all very speculative until we define sessions 2.0
with mechanisms to create new process sets. Achieving consensus on the new process set (content and naming) is on our list of semantic concerns in the Sessions WG for such functionality.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<ul style="margin-top:0cm" type="disc">
<li class="MsoListParagraph" style="margin-left:0cm;mso-list:l0 level1 lfo3"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Why do you want to see session_init as an immediate call? Shouldn’t this be able to setup things, which could
include allocations possible triggering longer resource negotiations in the OS? That call should not be in the critical path?<o:p></o:p></span></li></ul>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">[DAN] There is no requirement for MPI_Session_init to do anything complex, why do you want to add complexity where there is none and there is no need for it?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Cheers,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Martin<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt">--<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Prof. Dr. Martin Schulz, Chair of Computer Architecture and Parallel Systems<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Department of Informatics, TU-Munich, Boltzmannstraße 3, D-85748 Garching<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Member of the Board of Directors at the Leibniz Supercomputing Centre (LRZ)<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Email: <a href="mailto:schulzm@in.tum.de">
schulzm@in.tum.de</a><o:p></o:p></span></p>
</div>
</div>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span style="color:black">From: </span></b><span style="color:black">"Holmes, Daniel John" <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>><br>
<b>Date: </b>Thursday, 5. January 2023 at 02:50<br>
<b>To: </b>Martin Schulz <<a href="mailto:schulzm@in.tum.de">schulzm@in.tum.de</a>>, MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc: </b>"Pritchard Jr., Howard" <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject: </b>RE: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Hi Martin,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">This seems like the best reason so far for differentiating “immediate” as the term that refers to the stronger semantic.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">immediate = prohibited to delay its return – neither until (remote) progress nor until (remote) specific semantically-related MPI procedure call<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">local = permitted to delay its return until (remote) progress but not until (remote) specific semantically-related MPI procedure call<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">nonlocal = permitted to delay its return (remote) progress and/or until (remote) specific semantically-related MPI procedure call<o:p></o:p></span></p>
<div style="border:none;border-bottom:solid windowtext 1.0pt;padding:0cm 0cm 1.0pt 0cm">
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Permitting progress in an API that was intended to be “lightweight” is bad. Progress could mean that MPI_Session_init delays its return while it pushes a few GB of buffered-mode
send message data into a network – correctness is not affected, but performance expectations are. The canonical example is whether MPI_Comm_rank can delay its return while it does (remote) progress – currently “yes because local, but why would anyone implement
it that way? It should be immediate.” The recent debate about whether MPI_Put is nonlocal is relevant here – correctness allows us to say “it doesn’t matter if MPI_Put is nonlocal because the user cannot create a deadlock” but the performance expectations
of separating synchronisation from data movement instructions suggest that MPI_Put should be immediate, not even local, but definitely not nonlocal.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">I would be much happier with some kind of explicit “update the list of process set names” API that has whatever semantic is needed, rather than allowing (remote) progress in any
of the existing immediate APIs: “initialise a session”, “get the number of process set names”, “get the nth process set name”, and “make a group from this process set name”.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">The trouble is: what is the appropriate semantic for the “update the list of process set names” API? We already have MPI_Session_init that will give an updated list of process set
names if it is already different to previous lists given by previous sessions. We only need a “delay until something changes” semantic. What needs to change to satisfy this semantic? Is the first difference sufficient? Can/should the user specify what they
are looking for? Is any difference actually necessary (should it have a timeout, in case nothing changes)? Is it collective (over which group)? Those questions are currently answered by the mechanism(s) that could possibly cause change to the list of process
sets – spawn is an MPI operation, connect/accept are MPI operations, and so on. Initialising another session after one of these operations has completed will already get the up-to-date list of process sets (whether it changed because of the dynamic process
model operation or not). Any future proposal for an API that extends/modifies/prunes the list of process set names needs to have a semantic that permits the appropriate consensus in its implementation. The consensus is needed by operations like “give me more
resources”, “take these resources back”, etc – not by query functions.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Best wishes,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Dan.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt">From:</span></b><span lang="EN-US" style="font-size:11.0pt"> Martin Schulz <<a href="mailto:schulzm@in.tum.de">schulzm@in.tum.de</a>>
<br>
<b>Sent:</b> 04 January 2023 22:43<br>
<b>To:</b> Holmes, Daniel John <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>>; MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc:</b> Pritchard Jr., Howard <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject:</b> Re: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt;mso-fareast-language:EN-US">Hi Dan,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="DE" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">I fully agree with you on the difference between MPI and PVM and the need to keep things constant from the user’s perspective. They should not be impacted by this and
hence also the agreement that none of this can be non-local (as this will impact the user).<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">However, at some point the underlying runtime will have to come to a consensus and that may come after a user would expect it relative to a local procedure call. This
is not likely or probably not even possible with the current static scheme and implementation, but it will likely happen when we add more dynamic behavior.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">An example could be newly spawned processes and when they are ready and have established all their individual process set memberships. This could be, of course, pushed
off to the user who would need to ensure the right timing, but it may also be better to give the runtime some leeway – again, not in the sense of non-local, but in the sense of weak progress.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">This was actually my understanding of weak progress anyway – that any MPI routine could delay return for weak progress, but he hardened that only recently to operations
only. For the current set of MPI routines that actually does not make a difference in execution, but here we could – by accident – add a limiter for future implementations and this is what I would like to avoid by opening up the chance for these routines to
participate in progress.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Martin<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt">--<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Prof. Dr. Martin Schulz, Chair of Computer Architecture and Parallel Systems<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Department of Informatics, TU-Munich, Boltzmannstraße 3, D-85748 Garching<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Member of the Board of Directors at the Leibniz Supercomputing Centre (LRZ)<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Email: <a href="mailto:schulzm@in.tum.de">
schulzm@in.tum.de</a><o:p></o:p></span></p>
</div>
</div>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span style="color:black">From: </span></b><span style="color:black">"Holmes, Daniel John" <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>><br>
<b>Date: </b>Wednesday, 4. January 2023 at 10:17<br>
<b>To: </b>Martin Schulz <<a href="mailto:schulzm@in.tum.de">schulzm@in.tum.de</a>>, MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc: </b>"Pritchard Jr., Howard" <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject: </b>RE: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Hi Martin,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">MPI is not PVM. We do not wait to see which/how many processes start and join the group/process set before deciding on the membership of the group/process set. The names and the
membership of all (built-in/predefined) process sets are known a priori without coordination during the initialisation procedure call(s). Deviation from that membership (e.g. a process fails to start or fails to join up with the other processes) is a fault,
which will cause a failure (e.g. a collective operation cannot complete), which will manifest as an error. The process set still exists and a group can still be formed from it; the communicator creation procedure that uses that group will raise an error.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">For scenarios/implementations where additional process sets “appear” during the execution, those new process sets might not appear until all involved processes will see the same
new set name (depending on what the implementation can support); that might mean every involved process will have to have done some progress after the process set was created internally before any process will expose it to the user via MPI calls. That delay
must never happen for the built-in/predefined process sets, so we have no conflict or difficulty.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Best wishes,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US">Dan.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt">From:</span></b><span lang="EN-US" style="font-size:11.0pt"> Martin Schulz <<a href="mailto:schulzm@in.tum.de">schulzm@in.tum.de</a>>
<br>
<b>Sent:</b> 04 January 2023 19:43<br>
<b>To:</b> MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>>; Holmes, Daniel John <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>><br>
<b>Cc:</b> Pritchard Jr., Howard <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject:</b> Re: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Hi all,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">I agree with this interpretation – I always thought that was the original intent; non-local work should be able to be push off to the first communicator creation.
<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">The question about it being an operation and/or a local call is interesting, though – I tend to also see it the same as Dan, but is there a scenario in implementations
that may require some kind of progress in other MPI processes (e.g., to internally synchronize on process sets)? If so, would we have to classify at least some calls (perhaps only the query of the process sets) as (local) operations so we can mandate progress?
Or maybe “have to” is to harsh, but it would implementations to be more efficient?<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US">Martin<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div>
<div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt">--<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Prof. Dr. Martin Schulz, Chair of Computer Architecture and Parallel Systems<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Department of Informatics, TU-Munich, Boltzmannstraße 3, D-85748 Garching<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Member of the Board of Directors at the Leibniz Supercomputing Centre (LRZ)<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Email: <a href="mailto:schulzm@in.tum.de">
schulzm@in.tum.de</a><o:p></o:p></span></p>
</div>
</div>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;mso-fareast-language:EN-US"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="color:black">From: </span></b><span lang="EN-US" style="color:black">mpiwg-sessions <<a href="mailto:mpiwg-sessions-bounces@lists.mpi-forum.org">mpiwg-sessions-bounces@lists.mpi-forum.org</a>> on behalf of "Pritchard
Jr., Howard via mpiwg-sessions" <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Reply to: </b>MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Date: </b>Wednesday, 4. January 2023 at 09:30<br>
<b>To: </b>"Holmes, Daniel John" <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>>, MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc: </b>"Pritchard Jr., Howard" <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject: </b>Re: [mpiwg-sessions] [EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">HI Dan,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Yes that was my interpretation as well.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">We can discuss at our next meeting 1/9/23 if there’s time.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Howard<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span style="color:black">From: </span></b><span style="color:black">"Holmes, Daniel John" <<a href="mailto:daniel.john.holmes@intel.com">daniel.john.holmes@intel.com</a>><br>
<b>Date: </b>Wednesday, January 4, 2023 at 12:05 PM<br>
<b>To: </b>MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc: </b>"Pritchard Jr., Howard" <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject: </b>[EXTERNAL] RE: MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal"><span style="font-size:11.0pt">Hi Howard,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">It was always intended that MPI_Session_init was a local procedure. In fact, “initialise a session” is not even an MPI operation, so it doesn’t make sense for it to be expressed via a nonlocal procedure.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Further, it was intended that the nonlocal portion of the work done by MPI_Init that is eventually needed in the pure sessions pattern would be done during the first nonlocal procedure call in that pattern,
as follows:<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">MPI_Session_init // local – PMIx fence prohibited<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">MPI_Group_from_pset // local – PMIx fence prohibited<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">MPI_Comm_create_from_group // nonlocal – PMIx fence permitted, if needed<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">The nonlocal work should be unnecessary until the first nonlocal procedure call, so this should all work out fine (modulo some refactoring/debugging).<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Best wishes,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">Dan.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<div>
<div style="border:none;border-top:solid #E1E1E1 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal"><b><span lang="EN-US" style="font-size:11.0pt">From:</span></b><span lang="EN-US" style="font-size:11.0pt"> mpiwg-sessions <<a href="mailto:mpiwg-sessions-bounces@lists.mpi-forum.org">mpiwg-sessions-bounces@lists.mpi-forum.org</a>>
<b>On Behalf Of </b>Pritchard Jr., Howard via mpiwg-sessions<br>
<b>Sent:</b> 04 January 2023 18:32<br>
<b>To:</b> MPI Sessions working group <<a href="mailto:mpiwg-sessions@lists.mpi-forum.org">mpiwg-sessions@lists.mpi-forum.org</a>><br>
<b>Cc:</b> Pritchard Jr., Howard <<a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a>><br>
<b>Subject:</b> [mpiwg-sessions] MPI_Session_init semantics question/poll<o:p></o:p></span></p>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Hi All,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">First, Happy New Year!<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">I’ve got a question about the semantics of MPI_Session_init. In particular, I’d be interested in knowing people’s opinion on whether this function is nonlocal or local.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">We don’t have any text in the current version of the standard that states whether or not MPI_Session_init is a nonlocal operation.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">I’m considering options for handling this issue:
<a href="https://urldefense.com/v3/__https:/github.com/open-mpi/ompi/issues/11166__;!!Bt8fGhp8LhKGRg!CKPfJnVxgJ8KyXfu93oiW-q0IPGmpAtrBZo2vO6bAElAdqtSv6Xv6G48O6Hk2sxr3csENDhZPwUW0mA8_fi98l7TQUw$">
https://github.com/open-mpi/ompi/issues/11166</a> . It turns out that the way to properly resolve this issue depends on whether or not MPI_Session_init has local or nonlocal semantics.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">I had been working under the assumption that we had intended session initialization to be a local function, but considering how to resolve issue 11166 made me begin to question this assumption.<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Thanks for any ideas,<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt">Howard<o:p></o:p></span></p>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt"><o:p> </o:p></span></p>
<div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Arial",sans-serif;color:#0B1A8D"><br>
—</span><span lang="EN-US" style="font-size:11.0pt"><o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Arial",sans-serif;color:#0B1A8D"><o:p> </o:p></span></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" width="375" style="width:281.25pt;border-collapse:collapse">
<tbody>
<tr style="height:104.45pt">
<td width="78" valign="top" style="width:58.4pt;padding:0cm 5.4pt 0cm 5.4pt;height:104.45pt">
<p class="MsoNormal"><span style="font-size:9.0pt;font-family:"Arial",sans-serif"><img border="0" width="55" height="55" style="width:.5729in;height:.5729in" id="Picture_x0020_5" src="cid:image001.png@01D92438.55774A70" alt="signature_61897647"></span><span style="font-size:9.0pt;font-family:"Arial",sans-serif"><o:p></o:p></span></p>
</td>
<td width="297" valign="top" style="width:223.1pt;padding:0cm 5.4pt 0cm 5.4pt;height:104.45pt">
<p class="MsoNormal"><b><span style="font-family:"Arial",sans-serif;color:#0B1A8C">Howard Pritchard<o:p></o:p></span></b></p>
<p class="MsoNormal"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#545961">Research Scientist<o:p></o:p></span></b></p>
<p class="MsoNormal"><b><span style="font-size:10.0pt;font-family:"Arial",sans-serif;color:#545961">HPC-ENV<o:p></o:p></span></b></p>
<p class="MsoNormal"><span style="font-size:9.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C">Los Alamos National Laboratory<o:p></o:p></span></b></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C"><a href="mailto:howardp@lanl.gov">howardp@lanl.gov</a><o:p></o:p></span></b></p>
<p class="MsoNormal"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C"><o:p> </o:p></span></b></p>
<p class="MsoNormal"><a href="https://urldefense.com/v3/__https:/www.instagram.com/losalamosnatlab/__;!!Bt8fGhp8LhKGRg!CKPfJnVxgJ8KyXfu93oiW-q0IPGmpAtrBZo2vO6bAElAdqtSv6Xv6G48O6Hk2sxr3csENDhZPwUW0mA8_fi9Rgwox5A$"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C;text-decoration:none"><img border="0" width="27" height="27" style="width:.2812in;height:.2812in" id="Picture_x0020_4" src="cid:image002.png@01D92438.55774A70" alt="signature_1293224934"></span></b></a><a href="https://urldefense.com/v3/__https:/twitter.com/LosAlamosNatLab__;!!Bt8fGhp8LhKGRg!CKPfJnVxgJ8KyXfu93oiW-q0IPGmpAtrBZo2vO6bAElAdqtSv6Xv6G48O6Hk2sxr3csENDhZPwUW0mA8_fi9vR2-KGc$"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C;text-decoration:none"><img border="0" width="27" height="27" style="width:.2812in;height:.2812in" id="Picture_x0020_3" src="cid:image003.png@01D92438.55774A70" alt="signature_2498822630"></span></b></a><a href="https://urldefense.com/v3/__https:/www.linkedin.com/company/los-alamos-national-laboratory/__;!!Bt8fGhp8LhKGRg!CKPfJnVxgJ8KyXfu93oiW-q0IPGmpAtrBZo2vO6bAElAdqtSv6Xv6G48O6Hk2sxr3csENDhZPwUW0mA8_fi9_F2cjUc$"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C;text-decoration:none"><img border="0" width="27" height="27" style="width:.2812in;height:.2812in" id="Picture_x0020_2" src="cid:image004.png@01D92438.55774A70" alt="signature_1283032776"></span></b></a><a href="https://urldefense.com/v3/__https:/www.facebook.com/LosAlamosNationalLab/__;!!Bt8fGhp8LhKGRg!CKPfJnVxgJ8KyXfu93oiW-q0IPGmpAtrBZo2vO6bAElAdqtSv6Xv6G48O6Hk2sxr3csENDhZPwUW0mA8_fi95RavtTU$"><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C;text-decoration:none"><img border="0" width="27" height="27" style="width:.2812in;height:.2812in" id="Picture_x0020_1" src="cid:image005.png@01D92438.55774A70" alt="signature_3959178607"></span></b></a><b><span style="font-size:9.0pt;font-family:"Arial",sans-serif;color:#0B1A8C"><o:p></o:p></span></b></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal"><span lang="EN-US" style="font-size:11.0pt;font-family:"Arial",sans-serif"><o:p> </o:p></span></p>
</div>
</div>
<p class="MsoNormal"><span lang="EN-US"><o:p> </o:p></span></p>
</div>
</body>
</html>