%!PS-Adobe-2.0 %%Creator: dvips(k) 5.86 Copyright 1999 Radical Eye Software %%Title: arXiv:cs/0306048v1 [cs.DC] 11 Jun 2003 %%Pages: 10 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%DocumentFonts: Times-Bold Times-Roman Times-Italic Helvetica-Bold %%+ Courier %%EndComments %DVIPSWebPage: (www.radicaleye.com) %DVIPSCommandLine: dvips -R -P pk -K1 -z parallel-netcdf.dvi -o %DVIPSParameters: dpi=600, compressed, comments removed %DVIPSSource: TeX output 2017.12.13:0524 %%BeginProcSet: texc.pro /TeXDict 300 dict def TeXDict begin/N{def}def/B{bind def}N/S{exch}N/X{S N}B/A{dup}B/TR{translate}N/isls false N/vsize 11 72 mul N/hsize 8.5 72 mul N/landplus90{false}def/@rigin{isls{[0 landplus90{1 -1}{-1 1}ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{ landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR[ matrix currentmatrix{A A round sub abs 0.00001 lt{round}if}forall round exch round exch]setmatrix}N/@landscape{/isls true N}B/@manualfeed{ statusdict/manualfeed true put}B/@copies{/#copies X}B/FMat[1 0 0 -1 0 0] N/FBB[0 0 0 0]N/nn 0 N/IEn 0 N/ctr 0 N/df-tail{/nn 8 dict N nn begin /FontType 3 N/FontMatrix fntrx N/FontBBox FBB N string/base X array /BitMaps X/BuildChar{CharBuilder}N/Encoding IEn N end A{/foo setfont}2 array copy cvx N load 0 nn put/ctr 0 N[}B/sf 0 N/df{/sf 1 N/fntrx FMat N df-tail}B/dfs{div/sf X/fntrx[sf 0 0 sf neg 0 0]N df-tail}B/E{pop nn A definefont setfont}B/Cw{Cd A length 5 sub get}B/Ch{Cd A length 4 sub get }B/Cx{128 Cd A length 3 sub get sub}B/Cy{Cd A length 2 sub get 127 sub} B/Cdx{Cd A length 1 sub get}B/Ci{Cd A type/stringtype ne{ctr get/ctr ctr 1 add N}if}B/id 0 N/rw 0 N/rc 0 N/gp 0 N/cp 0 N/G 0 N/CharBuilder{save 3 1 roll S A/base get 2 index get S/BitMaps get S get/Cd X pop/ctr 0 N Cdx 0 Cx Cy Ch sub Cx Cw add Cy setcachedevice Cw Ch true[1 0 0 -1 -.1 Cx sub Cy .1 sub]/id Ci N/rw Cw 7 add 8 idiv string N/rc 0 N/gp 0 N/cp 0 N{ rc 0 ne{rc 1 sub/rc X rw}{G}ifelse}imagemask restore}B/G{{id gp get/gp gp 1 add N A 18 mod S 18 idiv pl S get exec}loop}B/adv{cp add/cp X}B /chg{rw cp id gp 4 index getinterval putinterval A gp add/gp X adv}B/nd{ /cp 0 N rw exit}B/lsh{rw cp 2 copy get A 0 eq{pop 1}{A 255 eq{pop 254}{ A A add 255 and S 1 and or}ifelse}ifelse put 1 adv}B/rsh{rw cp 2 copy get A 0 eq{pop 128}{A 255 eq{pop 127}{A 2 idiv S 128 and or}ifelse} ifelse put 1 adv}B/clr{rw cp 2 index string putinterval adv}B/set{rw cp fillstr 0 4 index getinterval putinterval adv}B/fillstr 18 string 0 1 17 {2 copy 255 put pop}for N/pl[{adv 1 chg}{adv 1 chg nd}{1 add chg}{1 add chg nd}{adv lsh}{adv lsh nd}{adv rsh}{adv rsh nd}{1 add adv}{/rc X nd}{ 1 add set}{1 add clr}{adv 2 chg}{adv 2 chg nd}{pop nd}]A{bind pop} forall N/D{/cc X A type/stringtype ne{]}if nn/base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{A A length 1 sub A 2 index S get sf div put }if put/ctr ctr 1 add N}B/I{cc 1 add D}B/bop{userdict/bop-hook known{ bop-hook}if/SI save N @rigin 0 0 moveto/V matrix currentmatrix A 1 get A mul exch 0 get A mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N/eop{ SI restore userdict/eop-hook known{eop-hook}if showpage}N/@start{ userdict/start-hook known{start-hook}if pop/VResolution X/Resolution X 1000 div/DVImag X/IEn 256 array N 2 string 0 1 255{IEn S A 360 add 36 4 index cvrs cvn put}for pop 65781.76 div/vsize X 65781.76 div/hsize X}N /p{show}N/RMat[1 0 0 -1 0 0]N/BDot 260 string N/Rx 0 N/Ry 0 N/V{}B/RV/v{ /Ry X/Rx X V}B statusdict begin/product where{pop false[(Display)(NeXT) (LaserWriter 16/600)]{A length product length le{A length product exch 0 exch getinterval eq{pop true exit}if}{pop}ifelse}forall}{false}ifelse end{{gsave TR -.1 .1 TR 1 1 scale Rx Ry false RMat{BDot}imagemask grestore}}{{gsave TR -.1 .1 TR Rx Ry scale 1 1 false RMat{BDot} imagemask grestore}}ifelse B/QV{gsave newpath transform round exch round exch itransform moveto Rx 0 rlineto 0 Ry neg rlineto Rx neg 0 rlineto fill grestore}B/a{moveto}B/delta 0 N/tail{A/delta X 0 rmoveto}B/M{S p delta add tail}B/b{S p tail}B/c{-4 M}B/d{-3 M}B/e{-2 M}B/f{-1 M}B/g{0 M} B/h{1 M}B/i{2 M}B/j{3 M}B/k{4 M}B/w{0 rmoveto}B/l{p -4 w}B/m{p -3 w}B/n{ p -2 w}B/o{p -1 w}B/q{p 1 w}B/r{p 2 w}B/s{p 3 w}B/t{p 4 w}B/x{0 S rmoveto}B/y{3 2 roll p a}B/bos{/SS save N}B/eos{SS restore}B end %%EndProcSet %%BeginProcSet: 8r.enc % File 8r.enc as of 2002-03-12 for PSNFSS 9 % % This is the encoding vector for Type1 and TrueType fonts to be used % with TeX. This file is part of the PSNFSS bundle, version 9 % % Authors: S. Rahtz, P. MacKay, Alan Jeffrey, B. Horn, K. Berry, W. Schmidt % % Idea is to have all the characters normally included in Type 1 fonts % available for typesetting. This is effectively the characters in Adobe % Standard Encoding + ISO Latin 1 + extra characters from Lucida + Euro. % % Character code assignments were made as follows: % % (1) the Windows ANSI characters are almost all in their Windows ANSI % positions, because some Windows users cannot easily reencode the % fonts, and it makes no difference on other systems. The only Windows % ANSI characters not available are those that make no sense for % typesetting -- rubout (127 decimal), nobreakspace (160), softhyphen % (173). quotesingle and grave are moved just because it's such an % irritation not having them in TeX positions. % % (2) Remaining characters are assigned arbitrarily to the lower part % of the range, avoiding 0, 10 and 13 in case we meet dumb software. % % (3) Y&Y Lucida Bright includes some extra text characters; in the % hopes that other PostScript fonts, perhaps created for public % consumption, will include them, they are included starting at 0x12. % % (4) Remaining positions left undefined are for use in (hopefully) % upward-compatible revisions, if someday more characters are generally % available. % % (5) hyphen appears twice for compatibility with both ASCII and Windows. % % (6) /Euro is assigned to 128, as in Windows ANSI % /TeXBase1Encoding [ % 0x00 (encoded characters from Adobe Standard not in Windows 3.1) /.notdef /dotaccent /fi /fl /fraction /hungarumlaut /Lslash /lslash /ogonek /ring /.notdef /breve /minus /.notdef % These are the only two remaining unencoded characters, so may as % well include them. /Zcaron /zcaron % 0x10 /caron /dotlessi % (unusual TeX characters available in, e.g., Lucida Bright) /dotlessj /ff /ffi /ffl /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef % very contentious; it's so painful not having quoteleft and quoteright % at 96 and 145 that we move the things normally found there down to here. /grave /quotesingle % 0x20 (ASCII begins) /space /exclam /quotedbl /numbersign /dollar /percent /ampersand /quoteright /parenleft /parenright /asterisk /plus /comma /hyphen /period /slash % 0x30 /zero /one /two /three /four /five /six /seven /eight /nine /colon /semicolon /less /equal /greater /question % 0x40 /at /A /B /C /D /E /F /G /H /I /J /K /L /M /N /O % 0x50 /P /Q /R /S /T /U /V /W /X /Y /Z /bracketleft /backslash /bracketright /asciicircum /underscore % 0x60 /quoteleft /a /b /c /d /e /f /g /h /i /j /k /l /m /n /o % 0x70 /p /q /r /s /t /u /v /w /x /y /z /braceleft /bar /braceright /asciitilde /.notdef % rubout; ASCII ends % 0x80 /Euro /.notdef /quotesinglbase /florin /quotedblbase /ellipsis /dagger /daggerdbl /circumflex /perthousand /Scaron /guilsinglleft /OE /.notdef /.notdef /.notdef % 0x90 /.notdef /.notdef /.notdef /quotedblleft /quotedblright /bullet /endash /emdash /tilde /trademark /scaron /guilsinglright /oe /.notdef /.notdef /Ydieresis % 0xA0 /.notdef % nobreakspace /exclamdown /cent /sterling /currency /yen /brokenbar /section /dieresis /copyright /ordfeminine /guillemotleft /logicalnot /hyphen % Y&Y (also at 45); Windows' softhyphen /registered /macron % 0xD0 /degree /plusminus /twosuperior /threesuperior /acute /mu /paragraph /periodcentered /cedilla /onesuperior /ordmasculine /guillemotright /onequarter /onehalf /threequarters /questiondown % 0xC0 /Agrave /Aacute /Acircumflex /Atilde /Adieresis /Aring /AE /Ccedilla /Egrave /Eacute /Ecircumflex /Edieresis /Igrave /Iacute /Icircumflex /Idieresis % 0xD0 /Eth /Ntilde /Ograve /Oacute /Ocircumflex /Otilde /Odieresis /multiply /Oslash /Ugrave /Uacute /Ucircumflex /Udieresis /Yacute /Thorn /germandbls % 0xE0 /agrave /aacute /acircumflex /atilde /adieresis /aring /ae /ccedilla /egrave /eacute /ecircumflex /edieresis /igrave /iacute /icircumflex /idieresis % 0xF0 /eth /ntilde /ograve /oacute /ocircumflex /otilde /odieresis /divide /oslash /ugrave /uacute /ucircumflex /udieresis /yacute /thorn /ydieresis ] def %%EndProcSet %%BeginProcSet: texps.pro TeXDict begin/rf{findfont dup length 1 add dict begin{1 index/FID ne 2 index/UniqueID ne and{def}{pop pop}ifelse}forall[1 index 0 6 -1 roll exec 0 exch 5 -1 roll VResolution Resolution div mul neg 0 0]FontType 0 ne{/Metrics exch def dict begin Encoding{exch dup type/integertype ne{ pop pop 1 sub dup 0 le{pop}{[}ifelse}{FontMatrix 0 get div Metrics 0 get div def}ifelse}forall Metrics/Metrics currentdict end def}{{1 index type /nametype eq{exit}if exch pop}loop}ifelse[2 index currentdict end definefont 3 -1 roll makefont/setfont cvx]cvx def}def/ObliqueSlant{dup sin S cos div neg}B/SlantFont{4 index mul add}def/ExtendFont{3 -1 roll mul exch}def/ReEncodeFont{CharStrings rcheck{/Encoding false def dup[ exch{dup CharStrings exch known not{pop/.notdef/Encoding true def}if} forall Encoding{]exch pop}{cleartomark}ifelse}if/Encoding exch def}def end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin/SDict 200 dict N SDict begin/@SpecialDefaults{/hs 612 N /vs 792 N/ho 0 N/vo 0 N/hsc 1 N/vsc 1 N/ang 0 N/CLIP 0 N/rwiSeen false N /rhiSeen false N/letter{}N/note{}N/a4{}N/legal{}N}B/@scaleunit 100 N /@hscale{@scaleunit div/hsc X}B/@vscale{@scaleunit div/vsc X}B/@hsize{ /hs X/CLIP 1 N}B/@vsize{/vs X/CLIP 1 N}B/@clip{/CLIP 2 N}B/@hoffset{/ho X}B/@voffset{/vo X}B/@angle{/ang X}B/@rwi{10 div/rwi X/rwiSeen true N}B /@rhi{10 div/rhi X/rhiSeen true N}B/@llx{/llx X}B/@lly{/lly X}B/@urx{ /urx X}B/@ury{/ury X}B/magscale true def end/@MacSetUp{userdict/md known {userdict/md get type/dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N/note{}N/legal{}N/od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{ itransform lineto}}{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{closepath}}pathforall newpath counttomark array astore/gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack} if}N/txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{ noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N/cp{pop pop showpage pm restore}N end}if}if}N/normalscale{ Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale }if 0 setgray}N/psfts{S 65781.76 div N}N/startTexFig{/psf$SavedState save N userdict maxlength dict begin/magscale true def normalscale currentpoint TR/psf$ury psfts/psf$urx psfts/psf$lly psfts/psf$llx psfts /psf$y psfts/psf$x psfts currentpoint/psf$cy X/psf$cx X/psf$sx psf$x psf$urx psf$llx sub div N/psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR/showpage{}N/erasepage{}N/setpagedevice{pop}N/copypage{}N/p 3 def @MacSetUp}N/doclip{psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N/endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin/SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count/ocount X/dcount countdictstack N} N/@setspecial{CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR}{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if/showpage{}N/erasepage{}N /setpagedevice{pop}N/copypage{}N newpath}N/@endspecial{count ocount sub{ pop}repeat countdictstack dcount sub{end}repeat grestore SpecialSave restore end}N/@defspecial{SDict begin}N/@fedspecial{end}B/li{lineto}B /rl{rlineto}B/rc{rcurveto}B/np{/SaveX currentpoint/SaveY X N 1 setlinecap newpath}N/st{stroke SaveX SaveY moveto}N/fil{fill SaveX SaveY moveto}N/ellipse{/endangle X/startangle X/yrad X/xrad X/savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet %%BeginProcSet: hps.pro /HPSdict 20 dict dup begin/braindeaddistill 50 def/rfch{dup length 1 sub 1 exch getinterval}bind def/splituri{dup(#)search{exch pop}{()exch} ifelse dup(file:)anchorsearch{pop exch pop 3 -1 roll pop false}{pop 3 -1 roll exch pop true}ifelse}bind def/lookuptarget{exch rfch dup /TargetAnchors where{pop TargetAnchors dup 3 -1 roll known{exch get true }{pop(target unknown:)print == false}ifelse}{pop pop (target dictionary unknown\012)print false}ifelse}bind def/savecount 0 def/stackstopped{count counttomark sub/savecount exch store stopped count savecount sub 1 sub dup 0 gt{{exch pop}repeat}{pop}ifelse}bind def /tempstring 256 string def/targetvalidate{1 index dup length 255 gt exch dup(/)search{pop pop pop exch pop true exch}{pop}ifelse cvn tempstring cvs token pop pop length 0 ne or not}bind def/targetdump-hook where{pop} {/targetdump-hook{dup mark exch gsave initmat setmatrix{{mark/Dest 4 2 roll targetvalidate{aload pop exch pop/Page 3 1 roll/View exch[exch /FitH exch]/DEST pdfmark}{cleartomark}ifelse}forall}stackstopped pop grestore}bind def}ifelse/baseurl{mark exch 1 dict dup 3 -1 roll/Base exch put/URI exch/DOCVIEW{pdfmark}stackstopped pop}bind def /externalhack systemdict/PDF known def/oldstyle true def/initmat matrix currentmatrix def/actiondict 2 dict dup/Subtype/URI put def /weblinkhandler{dup 3 1 roll mark 4 1 roll/Title 4 1 roll splituri 3 -1 roll dup length 0 gt{cvn/Dest exch 4 2 roll}{pop}ifelse{externalhack{ /HTTPFile exch}{actiondict dup 3 -1 roll/URI exch put/Action exch} ifelse}{externalhack{/HTTPFile exch}{/File exch/Action/GoToR}ifelse} ifelse counttomark 2 sub -1 roll aload pop/Rect 4 1 roll/Border 3 1 roll /Color exch oldstyle{/LNK}{/Subtype/Link/ANN}ifelse gsave initmat setmatrix{pdfmark}stackstopped grestore}bind def/externalhandler where{ pop}{/externalhandler{2 copy{weblinkhandler}exec{/externalhack externalhack not store 2 copy{weblinkhandler}exec{/externalhack externalhack not store/oldstyle false store 2 copy{weblinkhandler}exec{ (WARNING: external refs disabled\012)print/externalhandler{pop pop}bind store externalhandler}{pop pop}ifelse}{pop pop/externalhack externalhack not store}ifelse}{pop pop/externalhandler{weblinkhandler pop}bind store} ifelse}bind def}ifelse/pdfmnew{dup type/stringtype eq{externalhandler}{ exch dup rfch exch 3 -1 roll lookuptarget{mark 4 1 roll/Title 4 1 roll aload pop exch pop/Page 3 1 roll/View exch[exch/FitH exch]5 -1 roll aload pop/Rect 4 1 roll/Border 3 1 roll/Color exch/LNK gsave initmat setmatrix pdfmark grestore}{pop pop}ifelse}ifelse}bind def/pdfmold{dup type/stringtype eq{externalhandler}{exch dup rfch exch 3 -1 roll lookuptarget{mark 4 1 roll/Title 4 1 roll aload pop exch pop/Page 3 1 roll/View exch[exch/FitH exch]5 -1 roll aload pop pop 0 3 getinterval /Rect 3 1 roll/Border exch/LNK gsave initmat setmatrix pdfmark grestore} {pop pop}ifelse}ifelse}bind def/pdfm where{pop}{/pdfm /currentdistillerparams where{pop currentdistillerparams dup /CoreDistVersion known{/CoreDistVersion get}{0}ifelse dup braindeaddistill le{(WARNING: switching to old pdfm because version =) print ==/pdfmold}{pop/pdfmnew}ifelse load}{/pdfmark where{pop{dup type /stringtype eq{externalhandler}{2 copy mark 3 1 roll{pdfmnew} stackstopped{2 copy mark 3 1 roll{pdfmold}stackstopped{ (WARNING: pdfm disabled\012)print/pdfm{pop pop}store}{ (WARNING: new pdfm failed, switching to old pdfm\012)print/pdfm/pdfmold load store}ifelse}{/pdfm/pdfmnew load store}ifelse pop pop}ifelse}}{{ pop pop}}ifelse}ifelse bind def}ifelse end def %%EndProcSet TeXDict begin @defspecial /DvipsToPDF { 72.27 mul Resolution div } def /PDFToDvips { 72.27 div Resolution mul } def /HyperBorder { 1 PDFToDvips } def /H.V {pdf@hoff pdf@voff null} def /H.B {/Rect[pdf@llx pdf@lly pdf@urx pdf@ury]} def /H.S { currentpoint HyperBorder add /pdf@lly exch def dup DvipsToPDF /pdf@hoff exch def HyperBorder sub /pdf@llx exch def } def /H.L { 2 sub dup /HyperBasePt exch def PDFToDvips /HyperBaseDvips exch def currentpoint HyperBaseDvips sub /pdf@ury exch def /pdf@urx exch def } def /H.A { H.L currentpoint exch pop vsize 72 sub exch DvipsToPDF HyperBasePt sub sub /pdf@voff exch def } def /H.R { currentpoint HyperBorder sub /pdf@ury exch def HyperBorder add /pdf@urx exch def currentpoint exch pop vsize 72 sub exch DvipsToPDF sub /pdf@voff exch def } def systemdict /pdfmark known not {userdict /pdfmark systemdict /cleartomark get put} if @fedspecial end TeXDict begin 40258431 52099146 1000 600 600 (parallel-netcdf.dvi) @start /Fa 103[45 152[{ TeXBase1Encoding ReEncodeFont }1 74.7198 /Courier rf /Fb 133[29 33 33 50 33 37 21 29 29 1[37 37 37 54 21 33 1[21 37 37 21 33 37 33 37 37 9[62 1[54 1[37 2[46 54 1[62 42 1[33 25 54 1[46 46 54 50 1[46 6[25 37 1[37 37 37 37 37 37 37 37 21 19 25 19 4[25 35[37 37 2[{ TeXBase1Encoding ReEncodeFont }56 74.7198 /Times-Italic rf /Fc 107[33 33 24[33 37 37 54 37 37 21 29 25 1[37 37 37 58 21 37 21 21 37 37 25 33 37 33 37 33 3[25 1[25 46 1[54 71 54 54 46 42 50 54 42 54 54 66 46 54 29 25 54 54 42 46 54 50 50 54 1[33 4[21 37 37 37 37 37 37 37 37 37 37 21 19 25 19 2[25 25 25 35[42 42 2[{ TeXBase1Encoding ReEncodeFont }75 74.7198 /Times-Roman rf /Fd 134[42 3[46 28 32 37 46 1[42 46 1[23 2[23 2[28 37 46 37 46 42 16[51 65 1[78 3[32 65 1[51 1[60 2[60 12[42 1[42 42 1[23 21 43[46 2[{ TeXBase1Encoding ReEncodeFont }30 83.022 /Times-Bold rf /Fe 82[28 50[42 46 46 65 46 51 28 46 32 51 51 51 51 74 23 46 1[23 51 51 28 46 51 46 51 46 9[78 55 60 51 55 60 1[55 65 60 69 51 2[23 60 65 51 55 60 60 60 60 5[28 28 1[46 46 46 46 46 46 46 46 1[23 23 1[23 1[32 28 28 37[51 2[{ TeXBase1Encoding ReEncodeFont }63 83.022 /Helvetica-Bold rf /Ff 134[46 1[66 46 1[30 36 41 1[51 46 51 76 25 51 1[25 1[46 30 41 51 41 51 46 11[66 1[51 2[56 71 66 1[61 2[36 71 1[56 61 66 66 1[66 11[46 46 46 46 46 1[25 23 46[{ TeXBase1Encoding ReEncodeFont }40 91.3242 /Times-Bold rf /Fg 107[37 37 24[37 42 42 60 42 42 23 32 28 42 42 42 42 65 23 42 23 23 42 42 28 37 42 37 42 37 3[28 1[28 51 60 60 78 1[60 51 46 55 1[46 60 60 74 51 2[28 60 60 46 51 60 55 55 60 3[47 1[23 23 42 42 42 42 42 42 42 42 42 42 23 21 28 21 47 42 28 28 28 35[46 46 2[{ TeXBase1Encoding ReEncodeFont }76 83.022 /Times-Roman rf /Fh 133[32 37 37 55 37 42 23 32 32 1[42 42 42 60 23 37 1[23 42 42 23 37 42 37 42 42 9[69 2[46 3[51 60 1[69 3[28 60 1[51 1[60 55 1[51 11[42 5[23 21 28 21 40[42 42 2[{ TeXBase1Encoding ReEncodeFont }42 83.022 /Times-Italic rf /Fi 137[50 55 33 39 44 2[50 55 83 28 55 1[28 1[50 33 44 55 44 55 50 9[100 4[72 1[61 1[72 4[39 2[61 66 72 72 66 72 10[50 50 50 50 50 50 49[{ TeXBase1Encoding ReEncodeFont }35 99.6264 /Times-Bold rf %DVIPSBitmapFont: Fj cmsy10 12 2 /Fj 2 104 df<ED0FE015FF913803FC00EC0FE0EC3FC04A5A4AC7FC5C495AA2495AB3AD 495AA2495A131F495A495A01FEC8FCEA07F8EAFFE0A2EA07F8EA00FEEB7F806D7E6D7E13 0F6D7EA26D7EB3AD6D7EA26D7E806E7E6E7EEC0FE0EC03FC913800FFE0150F236479CA32 >102 D<12FEEAFFE0EA07F8EA00FEEB7F806D7E6D7E130F6D7EA26D7EB3AD6D7EA26D7E 806E7E6E7EEC0FE0EC03FC913800FFE0A2913803FC00EC0FE0EC3FC04A5A4AC7FC5C495A A2495AB3AD495AA2495A131F495A495A01FEC8FCEA07F8EAFFE048C9FC236479CA32>I E %EndDVIPSBitmapFont /Fk 134[44 1[66 44 1[28 39 39 1[50 50 50 72 28 2[28 50 50 1[44 2[50 50 11[72 1[50 4[66 83 55 6[61 72 66 1[61 20[25 44[{ TeXBase1Encoding ReEncodeFont }27 99.6264 /Times-Italic rf /Fl 134[50 1[72 50 50 28 39 33 1[50 50 50 78 28 50 28 28 50 50 1[44 50 44 50 44 9[94 2[61 1[66 5[61 1[39 2[72 3[66 1[72 92 17[25 33 25 44[{ TeXBase1Encoding ReEncodeFont }34 99.6264 /Times-Roman rf /Fm 139[40 1[53 2[60 66 100 33 2[33 66 60 40 53 1[53 1[60 13[66 2[73 93 5[47 93 1[73 1[86 86 1[86 6[40 10[33 1[40 42[66 2[{ TeXBase1Encoding ReEncodeFont }26 119.552 /Times-Bold rf end %%EndProlog %%BeginSetup %%Feature: *Resolution 600dpi TeXDict begin %%BeginPaperSize: Letter letter %%EndPaperSize HPSdict begin /TargetAnchors 0 dict dup begin end targetdump-hook def TeXDict begin %%EndSetup %%Page: 1 1 gsave %matrix defaultmatrix setmatrix 90 rotate /stampsize 20 def /Times-Roman findfont stampsize scalefont setfont currentfont /FontBBox get aload pop /pdf@top exch 1000 div stampsize mul def pop /pdf@bottom exch 1000 div stampsize mul def pop 252 -32 moveto currentpoint /pdf@lly exch pdf@bottom add def /pdf@llx exch 2 sub def 0.5 setgray (arXiv:cs/0306048v1 [cs.DC] 11 Jun 2003) show currentpoint /pdf@ury exch pdf@top add def /pdf@urx exch 2 add def /pdfmark where{pop}{userdict /pdfmark /cleartomark load put}ifelse [ /H /I /Border [0 0 1] /BS <</S/D/D[2 6]/W 1>> /Color [0 1 1] /Action << /Subtype /URI /URI (http://arxiv.org/abs/cs/0306048v1)>> /Subtype /Link /Rect[pdf@llx pdf@lly pdf@urx pdf@ury] /ANN pdfmark grestore 1 0 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a 0 0 a SDict begin [ /Title () /Subject () /Creator (LaTeX with hyperref package) /Author () /Producer (dvips + Distiller) /Keywords () /DOCINFO pdfmark end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.1) cvn H.B /DEST pdfmark end -114 0 a 368 374 a Fm(P)o(arallel)30 b(netCDF:)h(A)f(Scienti\002c)i(High-P)n(erf)m(ormance)e(I/O)f (Interface)961 690 y Fl(Jianwei)24 b(Li)149 b(W)-8 b(ei-k)o(eng)25 b(Liao)149 b(Alok)24 b(Choudhary)1088 806 y Fk(ECE)h(Department,)f (Northwestern)h(Univer)o(sity)912 922 y Fj(f)p Fl(jianwei,)f(wkliao,)g (choudhar)p Fj(g)p Fl(@ece.northwestern.edu)960 1155 y(Robert)h(Ross)149 b(Rajee)n(v)25 b(Thakur)149 b(W)l(illiam)24 b(Gropp)1047 1271 y Fk(MCS)i(Division,)d(Ar)l(gonne)h(National)g(Labor) o(atory)1225 1387 y Fj(f)p Fl(rross,)g(thakur)l(,)g(gropp)p Fj(g)p Fl(@mcs.anl.go)o(v)-114 1655 y SDict begin [ /Count -0 /Dest (section.1) cvn /Title (Introduction) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (section.2) cvn /Title (Related Work) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -3 /Dest (section.3) cvn /Title (NetCDF Background) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.3.1) cvn /Title (File Format) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.3.2) cvn /Title (Serial NetCDF API) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.3.3) cvn /Title (Using NetCDF in Parallel Environments) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -3 /Dest (section.4) cvn /Title (Parallel NetCDF) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.4.1) cvn /Title (Interface Design) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -2 /Dest (subsection.4.2) cvn /Title (Parallel Implementation) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsubsection.4.2.1) cvn /Title (Access to File Header) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsubsection.4.2.2) cvn /Title (Parallel I/O for Array Data) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.4.3) cvn /Title (Advantages and Disadvantages) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -2 /Dest (section.5) cvn /Title (Performance Evaluation) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.5.1) cvn /Title (Scalability Analysis) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (subsection.5.2) cvn /Title (FLASH I/O Performance) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Count -0 /Dest (section.6) cvn /Title (Conclusion and Future Work) /OUT pdfmark end -114 1655 a -114 1655 a SDict begin [ /Page 1 /View [ /Fit ] /PageMode /UseOutlines /DOCVIEW pdfmark end -114 1655 a -114 1655 a SDict begin [ {Catalog} << /ViewerPreferences << >> >> /PUT pdfmark end -114 1655 a -114 1655 a SDict begin H.S end -114 1655 a -114 1655 a SDict begin 12 H.A end -114 1655 a -114 1655 a SDict begin [ /View [/XYZ H.V] /Dest (Doc-Start) cvn H.B /DEST pdfmark end -114 1655 a 685 1738 a Fi(Abstract)-14 1943 y Fh(Dataset)34 b(stor)o(a)o(g)o(e)o(,)j(e)n(xc)o (hang)o(e)o(,)f(and)d(access)h(play)g(a)g(critical)-114 2043 y(r)l(ole)28 b(in)f(scienti\002c)g(applications.)44 b(F)-9 b(or)28 b(suc)o(h)f(purposes)f(netCDF)-114 2142 y(serves)f(as)f(a)g(portable)e(and)h(ef)o(\002cient)g(\002le)h(format)f (and)g(pr)l(o)o(gr)o(am-)-114 2242 y(ming)e(interface)o(,)h(whic)o(h)f (is)i(popular)d(in)i(numer)l(ous)f(scienti\002c)g(ap-)-114 2342 y(plication)c(domains.)23 b(Howe)o(ver)-9 b(,)19 b(the)f(original)g(interface)g(does)g(not)-114 2441 y(pr)l(o)o(vide)i (a)g(ef)o(\002cient)g(mec)o(hanism)f(for)i(par)o(allel)f(data)f(stor)o (a)o(g)o(e)h(and)-114 2541 y(access.)-14 2647 y(In)c(this)h(work,)g(we) g(pr)m(esent)g(a)f(ne)o(w)h(par)o(allel)e(interface)h(for)h(writ-)-114 2747 y(ing)k(and)f(r)m(eading)g(netCDF)h(datasets.)27 b(This)22 b(interface)f(is)h(derived)-114 2846 y(with)34 b(minimum)g(c)o(hang)o(es)e(fr)l(om)j(the)e(serial)i(netCDF)f (interface)-114 2946 y(b)n(ut)21 b(de\002nes)f(semantics)h(for)h(par)o (allel)e(access)h(and)f(is)i(tailor)m(ed)f(for)-114 3045 y(high)h(performance)o(.)31 b(The)23 b(underlying)e(par)o(allel)h(I/O)h (is)g(ac)o(hie)o(ved)-114 3145 y(thr)l(ough)e(MPI-IO,)g(allowing)g(for) i(dr)o(amatic)e(performance)g(gains)-114 3245 y(thr)l(ough)29 b(the)h(use)h(of)f(collective)g(I/O)g(optimizations.)55 b(W)-8 b(e)31 b(com-)-114 3344 y(par)m(e)k(the)g(implementation)f(str)o (ate)m(gies)h(with)h(HDF5)f(and)g(ana-)-114 3444 y(lyze)19 b(both.)24 b(Our)19 b(tests)g(indicate)f(pr)l(o)o(gr)o(amming)f(con)m (venience)f(and)-114 3544 y(signi\002cant)k(I/O)i(performance)f(impr)l (o)o(vement)g(with)i(this)f(par)o(allel)-114 3643 y(netCDF)e(interface) o(.)-114 3883 y SDict begin H.S end -114 3883 a -114 3883 a SDict begin 12 H.A end -114 3883 a -114 3883 a SDict begin [ /View [/XYZ H.V] /Dest (section.1) cvn H.B /DEST pdfmark end -114 3883 a 105 x Fi(1)99 b(Intr)n(oduction)-14 4216 y Fg(Scientists)36 b(ha)n(v)o(e)f(recognized)e(the)j(importance)d (of)j(portable)-114 4316 y(and)24 b(ef)n(\002cient)g(mechanisms)f(for)h (storing)f(lar)o(ge)g(datasets)i(created)-114 4416 y(and)17 b(used)h(by)g(their)f(applications.)23 b(The)18 b(Netw)o(ork)f(Common)g (Data)-114 4515 y(F)o(orm)32 b(\(netCDF\))h([)478 4517 y SDict begin H.S end 478 4517 a -2 x Fg(9)519 4459 y SDict begin H.R end 519 4459 a 519 4515 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ReDa90) cvn H.B /ANN pdfmark end 519 4515 a Fg(,)573 4516 y SDict begin H.S end 573 4516 a -1 x Fg(8)615 4459 y SDict begin H.R end 615 4459 a 615 4515 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RDED97) cvn H.B /ANN pdfmark end 615 4515 a Fg(])g(is)h(one)e(such)h (mechanism)f(used)g(by)h(a)-114 4615 y(number)18 b(of)i(applications.) -14 4721 y(NetCDF)50 b(intends)g(to)g(pro)o(vide)e(a)i(common)e(data)i (access)-114 4821 y(method)30 b(for)g(atmospheric)f(science)i (applications)f(to)h(deal)g(with)-114 4920 y(a)e(v)n(ariety)e(of)h (data)f(types)h(that)h(encompass)e(single-point)f(obser)n(-)-114 5020 y(v)n(ations,)d(time)h(series,)h(re)o(gularly)c(spaced)i(grids,)g (and)g(satellite)i(or)-114 5119 y(radar)19 b(images)g([)360 5120 y SDict begin H.S end 360 5120 a -1 x Fg(8)401 5063 y SDict begin H.R end 401 5063 a 401 5119 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RDED97) cvn H.B /ANN pdfmark end 401 5119 a Fg(].)25 b(T)-7 b(oday)19 b(se)n(v)o(eral)f(or)o (ganizations)f(ha)n(v)o(e)i(adopted)-114 5219 y(netCDF)i(as)g(a)f(data) g(access)h(standard)e([)1039 5221 y SDict begin H.S end 1039 5221 a -2 x Fg(19)1122 5162 y SDict begin H.R end 1122 5162 a 1122 5219 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.WINU) cvn H.B /ANN pdfmark end 1122 5219 a Fg(].)-14 5325 y(The)26 b(netCDF)h(design)e(consists)i(of)f(both)g(a)h (portable)e(\002le)i(for)n(-)2042 1738 y(mat)f(and)f(an)h(easy-to-use)e (application)g(programming)e(interf)o(ace)2042 1837 y(\(API\))i(for)g (storing)g(and)g(retrie)n(ving)e(netCDF)j(\002les)h(across)e(multi-) 2042 1937 y(ple)17 b(platforms.)22 b(More)16 b(and)g(more)f (scienti\002c)i(applications)e(choose)2042 2037 y(netCDF)32 b(as)g(their)f(output)g(\002le)h(format.)57 b(While)32 b(these)g(applica-)2042 2136 y(tions)18 b(become)f(computational)f(and) h(data)h(intensi)n(v)o(e,)f(the)o(y)h(tend)f(to)2042 2236 y(be)26 b(parallelized)g(on)f(high-performance)d(computers.)42 b(Hence,)27 b(it)2042 2335 y(is)17 b(highly)e(desirable)h(to)g(ha)n(v)o (e)g(an)g(ef)n(\002cient)f(parallel)h(programming)2042 2435 y(interf)o(ace)g(to)g(the)h(netCDF)g(\002les.)24 b(Unfortunately)-5 b(,)14 b(the)j(original)e(de-)2042 2535 y(sign)24 b(of)f(netCDF)g(interf)o(ace)g(is)h(pro)o(ving)d (inadequate)g(for)i(parallel)2042 2634 y(applications)j(because)g(of)g (its)h(lack)g(of)f(a)h(parallel)f(access)i(mech-)2042 2734 y(anism.)56 b(In)30 b(particular)m(,)h(there)f(is)h(no)f(support)f (for)g(concurrently)2042 2834 y(writing)21 b(to)g(a)h(netCDF)f(\002le.) 29 b(Hence,)21 b(parallel)g(applications)e(oper)n(-)2042 2933 y(ating)26 b(on)f(netCDF)i(\002les)f(must)g(serialize)g(access.)43 b(T)m(raditionally)-5 b(,)2042 3033 y(parallel)29 b(applications)e (write)i(to)g(netCDF)g(\002les)h(through)d(one)h(of)2042 3132 y(the)d(allocated)e(process)h(which)g(easily)h(becomes)f(a)g (performance)2042 3232 y(bottleneck.)31 b(The)22 b(serial)h(I/O)g (access)g(is)g(both)f(slo)n(w)h(and)f(cumber)n(-)2042 3332 y(some)e(to)h(the)f(application)f(programmer)-5 b(.)2142 3432 y(T)e(o)29 b(f)o(acilitate)g(parallel)f(I/O)h (operations,)f(we)h(ha)n(v)o(e)g(de\002ned)e(a)2042 3531 y(parallel)22 b(API)g(for)g(concurrently)d(accessing)i(netCDF)i (\002les.)31 b(W)m(ith)2042 3631 y(minimum)21 b(changes)f(to)i(the)g (names)f(and)h(ar)o(gument)d(lists,)k(this)g(in-)2042 3731 y(terf)o(ace)e(maintains)g(the)h(look)f(and)g(feel)h(of)f(the)h (serial)g(netCDF)g(in-)2042 3830 y(terf)o(ace)31 b(while)g(the)g (implementation)d(underneath)h(incorporates)2042 3930 y(well-kno)n(wn)e(parallel)i(I/O)h(techniques)d(such)i(as)h(collecti)n (v)o(e)e(I/O)2042 4029 y(to)23 b(allo)n(w)g(high-performance)18 b(data)k(access.)33 b(W)-7 b(e)24 b(implement)d(this)2042 4129 y(w)o(ork)i(on)h(top)f(of)g(MPI-IO,)g(which)g(is)i(speci\002ed)e (by)h(MPI-2)f(stan-)2042 4229 y(dard)34 b([)2253 4230 y SDict begin H.S end 2253 4230 a -1 x Fg(3)2294 4172 y SDict begin H.R end 2294 4172 a 2294 4229 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.GrLT99) cvn H.B /ANN pdfmark end 2294 4229 a Fg(,)2350 4230 y SDict begin H.S end 2350 4230 a -1 x Fg(7)2392 4173 y SDict begin H.R end 2392 4173 a 2392 4229 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.Mess97) cvn H.B /ANN pdfmark end 2392 4229 a Fg(,)2447 4229 y SDict begin H.S end 2447 4229 a Fg(2)2489 4172 y SDict begin H.R end 2489 4172 a 2489 4229 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.GLDS96) cvn H.B /ANN pdfmark end 2489 4229 a Fg(])h(and)f(is)h(freely)f(a)n(v)n(ailable)g (on)g(most)h(platforms.)2042 4328 y(Since)i(MPI)g(has)h(become)d(de)i (f)o(acto)g(parallel)f(mechanism)g(for)2042 4428 y(communication)17 b(and)h(I/O)i(on)f(most)g(parallel)g(en)m(vironments,)d(this)2042 4528 y(approach)i(is)j(portable)e(across)h(dif)n(ferent)f(platforms.) 2142 4628 y(Hierarchical)e(Data)h(F)o(ormat)f(v)o(ersion)f(5)i (\(HDF5\))f([)3633 4629 y SDict begin H.S end 3633 4629 a -1 x Fg(5)3675 4571 y SDict begin H.R end 3675 4571 a 3675 4628 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.HDF5) cvn H.B /ANN pdfmark end 3675 4628 a Fg(])h(also)g(pro-)2042 4727 y(vides)i(a)h(portable)e(\002le)i(format)e(and)h(programming)c (interf)o(aces)k(for)2042 4827 y(storing)c(multidimensional)f(arrays)i (together)e(with)j(ancillary)e(data)2042 4926 y(in)22 b(a)h(single)e(\002le.)31 b(It)22 b(supports)f(parallel)g(I/O)h(and)f (its)i(implementa-)2042 5026 y(tion)32 b(is)i(also)e(b)n(uilt)h(on)f (top)g(of)g(MPI-IO.)g(Ho)n(we)n(v)o(er)m(,)h(the)f(HDF5)2042 5126 y(API)i(is)h(too)f(\003e)o(xible)f(and)g(cumbrous)f(to)i(become)f (an)h(easy-to-)2042 5225 y(use)25 b(standard)f(since)h(it)g(adds)g (more)f(programming)d(features)j(and)2042 5325 y(completely)32 b(re-designs)g(the)h(API)g(from)f(its)j(pre)n(vious)c(v)o(ersion.)1928 5574 y(1)p eop %%Page: 2 2 2 1 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.2) cvn H.B /DEST pdfmark end -114 0 a 83 x Fg(Our)27 b(parallel)g(netCDF)h(interf)o(ace,) g(on)e(the)i(other)e(hand,)i(is)g(more)-114 183 y(concise,)i(closer)f (to)g(the)g(original)f(API,)h(and)f(goes)h(much)e(closer)-114 282 y(to)d(MPI-IO)f(interf)o(ace,)h(which)f(introduces)f(less)j(o)o(v)o (erhead)c(while)-114 382 y(pro)o(viding)c(more)i(optimization)f (opportunities)g(for)h(performance)-114 482 y(enhancement.)27 b(The)21 b(goal)g(of)h(this)g(w)o(ork)f(is)h(to)g(mak)o(e)f(the)h (parallel)-114 581 y(netCDF)j(interf)o(ace)f(a)i(data)f(access)g (standard)f(for)g(parallel)g(scien-)-114 681 y(ti\002c)d(applications.) -14 782 y(W)-7 b(e)29 b(run)e(a)i(couple)e(of)h(benchmarks)e(using)h (parallel)h(netCDF)-114 881 y(and)c(parallel)f(HDF5,)i(e)o(xploring)d (both)h(arti\002cially)h(made)g(access)-114 981 y(patterns)e(from)f (our)g(o)n(wn)h(benchmark)e(and)h(the)i(ones)e(from)h(a)g(real)-114 1081 y(application)k(called)i(FLASH)h([)830 1081 y SDict begin H.S end 830 1081 a Fg(1)871 1024 y SDict begin H.R end 871 1024 a 871 1081 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.FORT00) cvn H.B /ANN pdfmark end 871 1081 a Fg(].)48 b(In)28 b(our)f(e)o(xperiments,)h(paral-)-114 1180 y(lel)g(netCDF)f(brings)f(signi\002cant)h(I/O)g(impro)o(v)o(ement) d(and)i(sho)n(ws)-114 1280 y(better)h(performance)d(than)i(parallel)h (HDF5)g(in)g(the)h(FLASH)f(I/O)-114 1380 y(benchmark)18 b([)304 1381 y SDict begin H.S end 304 1381 a -1 x Fg(18)386 1323 y SDict begin H.R end 386 1323 a 386 1380 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.FLASHIO) cvn H.B /ANN pdfmark end 386 1380 a Fg(].)-14 1481 y(The)k(rest)g(of)g(this)h (paper)f(is)h(or)o(ganized)c(as)k(follo)n(ws.)31 b(Section)1813 1481 y SDict begin H.S end 1813 1481 a Fg(2)1855 1424 y SDict begin H.R end 1855 1424 a 1855 1481 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (section.2) cvn H.B /ANN pdfmark end 1855 1481 a -114 1580 a Fg(re)n(vie)n(ws)c(some)g(related)g(w)o (ork.)45 b(Section)1139 1581 y SDict begin H.S end 1139 1581 a -1 x Fg(3)1181 1524 y SDict begin H.R end 1181 1524 a 1181 1580 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (section.3) cvn H.B /ANN pdfmark end 1181 1580 a 28 w Fg(presents)27 b(the)g(design)-114 1680 y(background)17 b(of)k(netCDF)g(and)g(points)f (out)h(its)h(potential)e(usage)g(in)-114 1780 y(parallel)d (scienti\002c)g(applications.)23 b(The)16 b(design)h(and)f(implementa-) -114 1879 y(tion)g(of)g(our)f(parallel)h(netCDF)g(is)h(described)e(in)h (Section)1544 1879 y SDict begin H.S end 1544 1879 a Fg(4)1585 1823 y SDict begin H.R end 1585 1823 a 1585 1879 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (section.4) cvn H.B /ANN pdfmark end 1585 1879 a Fg(.)25 b(Exper)n(-)-114 1979 y(imental)20 b(performance)e(results)j(are)g(gi)n(v)o(en)e(in)i (Section)1516 1980 y SDict begin H.S end 1516 1980 a -1 x Fg(5)1558 1922 y SDict begin H.R end 1558 1922 a 1558 1979 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (section.5) cvn H.B /ANN pdfmark end 1558 1979 a Fg(.)27 b(Section)-114 2079 y SDict begin H.S end -114 2079 a -1 x Fg(6)-72 2022 y SDict begin H.R end -72 2022 a -72 2078 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (section.6) cvn H.B /ANN pdfmark end -72 2078 a 20 w Fg(concludes)19 b(the)h(paper)-5 b(.)-114 2214 y SDict begin H.S end -114 2214 a -114 2214 a SDict begin 12 H.A end -114 2214 a -114 2214 a SDict begin [ /View [/XYZ H.V] /Dest (section.2) cvn H.B /DEST pdfmark end -114 2214 a 100 x Fi(2)99 b(Related)26 b(W)-7 b(ork)-14 2533 y Fg(Considerable)39 b(research)i(has)g(been)g(done)f (on)h(data)g(access)-114 2632 y(for)29 b(scienti\002c)g(applications.) 51 b(The)29 b(w)o(ork)f(has)i(focused)e(on)g(data)-114 2732 y(I/O)f(performance)c(and)j(data)h(management)d(con)m(v)o (enience.)41 b(T)-7 b(w)o(o)-114 2831 y(projects,)24 b(MPI-IO)g(and)f(HDF)-7 b(,)25 b(are)f(most)g(closely)g(related)g(to)g (our)-114 2931 y(research.)-14 3032 y(MPI-IO)19 b(is)i(a)g(parallel)f (I/O)g(interf)o(ace)g(speci\002ed)f(in)i(the)f(MPI-2)-114 3132 y(standard.)51 b(It)29 b(is)h(implemented)d(and)i(used)g(on)g(a)g (wide)g(range)g(of)-114 3231 y(platforms.)c(The)20 b(most)g(popular)f (implementation,)f(R)m(OMIO)i([)1744 3232 y SDict begin H.S end 1744 3232 a -1 x Fg(17)1827 3175 y SDict begin H.R end 1827 3175 a 1827 3231 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.TRLG02) cvn H.B /ANN pdfmark end 1827 3231 a Fg(])-114 3331 y(is)32 b(implemented)d(portably)g(on)h(top)h(of) g(an)f(abstract)h(I/O)g(de)n(vice)-114 3431 y(layer)18 b([)99 3431 y SDict begin H.S end 99 3431 a Fg(14)182 3374 y SDict begin H.R end 182 3374 a 182 3431 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThGL96) cvn H.B /ANN pdfmark end 182 3431 a Fg(,)221 3432 y SDict begin H.S end 221 3432 a -1 x Fg(16)304 3374 y SDict begin H.R end 304 3374 a 304 3431 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThGL99b) cvn H.B /ANN pdfmark end 304 3431 a Fg(])h(that)f(enables)g (portability)f(to)h(ne)n(w)g(underlying)e(I/O)-114 3530 y(systems.)61 b(One)33 b(of)e(the)i(most)f(important)e(features)i(in)g (R)m(OMIO)-114 3630 y(is)e(collecti)n(v)o(e)f(I/O)g(operations,)h (which)f(adopt)f(a)i(tw)o(o-phase)e(I/O)-114 3730 y(strate)o(gy)16 b([)193 3730 y SDict begin H.S end 193 3730 a Fg(11)276 3673 y SDict begin H.R end 276 3673 a 276 3730 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RoBC93) cvn H.B /ANN pdfmark end 276 3730 a Fg(,)314 3730 y SDict begin H.S end 314 3730 a Fg(12)397 3673 y SDict begin H.R end 397 3673 a 397 3730 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.TBCP94) cvn H.B /ANN pdfmark end 397 3730 a Fg(,)435 3731 y SDict begin H.S end 435 3731 a -1 x Fg(13)518 3673 y SDict begin H.R end 518 3673 a 518 3730 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThCh96) cvn H.B /ANN pdfmark end 518 3730 a Fg(,)556 3731 y SDict begin H.S end 556 3731 a -1 x Fg(15)639 3673 y SDict begin H.R end 639 3673 a 639 3730 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThGL99) cvn H.B /ANN pdfmark end 639 3730 a Fg(])h(and)f(impro)o(v)o(e)f (the)i(parallel)f(I/O)h(perfor)n(-)-114 3829 y(mance)i(by)g (signi\002cantly)g(reducing)e(the)j(number)e(of)h(I/O)h(requests)-114 3929 y(that)27 b(w)o(ould)f(otherwise)g(result)h(in)g(man)o(y)e(small,) k(noncontiguous)-114 4028 y(I/O)j(requests.)58 b(Ho)n(we)n(v)o(er)m(,) 32 b(MPI-IO)e(reads)h(and)g(writes)h(data)f(in)-114 4128 y(a)26 b(ra)o(w)g(format)f(without)h(pro)o(viding)d(an)o(y)i (functionality)f(to)i(ef)n(fec-)-114 4228 y(ti)n(v)o(ely)j(manage)e (the)j(associated)f(metadata.)51 b(Nor)28 b(does)h(it)h(guar)n(-)-114 4327 y(antee)h(data)h(portability)-5 b(,)32 b(thereby)f(making)f(it)i (incon)m(v)o(enient)d(for)-114 4427 y(scientists)h(to)e(or)o(ganize,)g (transfer)m(,)h(and)f(share)h(their)f(application)-114 4527 y(data.)-14 4628 y(HDF)c(is)g(a)g(\002le)g(format)f(and)g(softw)o (are,)h(de)n(v)o(eloped)d(at)j(NCSA,)-114 4727 y(for)g(storing,)i (retrie)n(ving,)e(analyzing,)g(visualizing,)h(and)g(con)m(v)o(ert-)-114 4827 y(ing)38 b(scienti\002c)g(data.)78 b(The)38 b(most)g(popular)e(v)o (ersions)h(of)h(HDF)-114 4926 y(are)32 b(HDF4)g([)288 4926 y SDict begin H.S end 288 4926 a Fg(4)329 4870 y SDict begin H.R end 329 4870 a 329 4926 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.HDF4) cvn H.B /ANN pdfmark end 329 4926 a Fg(])g(and)f(HDF5)h([)809 4927 y SDict begin H.S end 809 4927 a -1 x Fg(5)850 4870 y SDict begin H.R end 850 4870 a 850 4926 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.HDF5) cvn H.B /ANN pdfmark end 850 4926 a Fg(].)61 b(The)31 b(design)g(goal)h(of)f(HDF4)-114 5026 y(is)d(mainly)f(to)h(deal)f(with)h(sequential)f(data)g(access)h (and)f(its)i(APIs)-114 5126 y(are)c(consistent)g(with)g(its)h(earlier)f (v)o(ersions.)38 b(On)25 b(the)g(other)g(hand,)-114 5225 y(HDF5)d(is)h(a)g(major)e(re)n(vision)h(in)g(which)g(its)h(APIs)g(are)f (completely)-114 5325 y(re-designed.)53 b(Both)30 b(v)o(ersions)f (store)h(multidimensional)e(arrays)2042 83 y(together)22 b(with)g(ancillary)g(data)h(in)g(portable,)e(self-describing)g(\002le) 2042 183 y(formats.)52 b(The)29 b(support)f(for)h(parallel)f(data)i (access)g(in)f(HDF5)h(is)2042 282 y(b)n(uilt)25 b(on)g(top)f(of)h (MPI-IO,)f(which)h(ensures)f(its)i(portability)d(since)2042 382 y(MPI-IO)32 b(has)h(become)e(a)i(de)g(f)o(acto)f(standard)g(for)f (parallel)i(I/O.)2042 482 y(Ho)n(we)n(v)o(er)m(,)i(the)f(f)o(act)g (that)f(HDF5)h(\002le)g(format)f(is)i(not)e(compati-)2042 581 y(ble)21 b(with)g(HDF4)f(can)g(be)h(incon)m(v)o(enient)c(for)j(e)o (xisting)g(HDF4)h(pro-)2042 681 y(grammers)k(to)h(migrate)f(their)h (applications)e(to)i(HDF5.)43 b(Further)n(-)2042 780 y(more,)28 b(HDF5)f(adds)f(se)n(v)o(eral)g(ne)n(w)h(features,)g(such)g (as)g(hierarchi-)2042 880 y(cal)d(\002le)h(structure,)e(to)h(describe)f (more)g(metadata,)h(b)n(ut)g(it)h(also)f(in-)2042 980 y(creases)34 b(the)f(dif)n(\002culties)g(for)g(the)g(implementation)e (of)i(parallel)2042 1079 y(data)f(access)g(underneath.)58 b(And)31 b(the)h(o)o(v)o(erhead)d(in)m(v)n(olv)o(ed)h(may)2042 1179 y(mak)o(e)21 b(HDF5)h(perform)d(much)i(w)o(orse)g(than)g(its)i (underlying)c(MPI-)2042 1279 y(IO.)j(By)g(using)g(a)g(number)e(of)i (scienti\002c)g(applications,)f(this)h(prob-)2042 1378 y(lem)f(is)g(addressed)e(in)h([)2725 1379 y SDict begin H.S end 2725 1379 a -1 x Fg(6)2766 1322 y SDict begin H.R end 2766 1322 a 2766 1378 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.LLCT02) cvn H.B /ANN pdfmark end 2766 1378 a Fg(,)2808 1379 y SDict begin H.S end 2808 1379 a -1 x Fg(10)2891 1322 y SDict begin H.R end 2891 1322 a 2891 1378 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RNCZ01) cvn H.B /ANN pdfmark end 2891 1378 a Fg(].)2042 1506 y SDict begin H.S end 2042 1506 a 2042 1506 a SDict begin 12 H.A end 2042 1506 a 2042 1506 a SDict begin [ /View [/XYZ H.V] /Dest (section.3) cvn H.B /DEST pdfmark end 2042 1506 a 103 x Fi(3)100 b(NetCDF)25 b(Backgr)n(ound)2142 1824 y Fg(NetCDF)31 b(is)h(an)e(abstraction)g(that)g(supports)g(a)h (vie)n(w)f(of)h(data)2042 1924 y(as)g(a)g(collection)e(of)h (self-describing,)g(portable,)h(array-oriented)2042 2023 y(objects)25 b(that)h(can)f(be)g(accessed)g(through)e(a)j(simple)f (interf)o(ace.)39 b(It)2042 2123 y(de\002nes)23 b(a)h(\002le)f(format)f (as)i(well)f(as)h(a)g(set)g(of)e(programming)e(inter)n(-)2042 2222 y(f)o(aces)29 b(for)f(storing)g(and)h(retrie)n(ving)e(data)h(in)h (the)g(form)f(of)g(arrays)2042 2322 y(in)g(netCDF)h(\002les.)48 b(W)-7 b(e)29 b(\002rst)g(describe)e(the)h(netCDF)g(\002le)h(format) 2042 2422 y(and)c(its)h(serial)g(API)f(and)g(then)g(consider)f(v)n (arious)g(approaches)f(to)2042 2521 y(access)e(netCDF)g(\002les)g(in)f (parallel)g(computing)e(en)m(vironments.)2042 2646 y SDict begin H.S end 2042 2646 a 2042 2646 a SDict begin 12 H.A end 2042 2646 a 2042 2646 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.3.1) cvn H.B /DEST pdfmark end 2042 2646 a 82 x Ff(3.1)91 b(File)23 b(F)n(ormat)2142 2934 y Fg(NetCDF)k(stores)f(data)g(in)h(an)f(array-oriented)d(dataset,)k (which)2042 3034 y(contains)g(dimensions,)i(v)n(ariables,)f(and)f (attrib)n(utes.)48 b(Physically)-5 b(,)2042 3133 y(the)22 b(dataset)g(\002le)h(is)f(di)n(vided)f(into)g(tw)o(o)h(parts:)29 b(\002le)23 b(header)d(and)i(ar)n(-)2042 3233 y(ray)f(data.)26 b(The)21 b(header)f(contains)g(all)h(information)d(\(or)j(metadata\)) 2042 3332 y(about)30 b(dimensions,)i(attrib)n(utes,)i(and)c(v)n (ariables)g(e)o(xcept)g(for)h(the)2042 3432 y(v)n(ariable)e(data)h (itself,)i(while)e(the)g(data)g(part)f(contains)h(arrays)f(of)2042 3532 y(v)n(ariable)19 b(v)n(alues)h(\(or)g(ra)o(w)g(data\).)2142 3631 y(The)g(netCDF)h(\002le)h(header)d(\002rst)i(de\002nes)g(a)g (number)d(of)j(dimen-)2042 3731 y(sions,)e(each)f(with)h(a)g(name)f (and)g(a)h(length.)24 b(These)18 b(dimensions)g(are)2042 3831 y(used)g(to)g(de\002ne)g(the)g(shapes)g(of)g(v)n(ariables)f(in)h (the)g(dataset.)25 b(One)18 b(di-)2042 3930 y(mension)e(can)g(be)g (unlimited)g(and)f(is)j(used)e(as)h(the)g(most)f(signi\002cant)2042 4030 y(dimension)j(\(record)f(dimension\))h(for)g(gro)n(wing-size)f(v)n (ariables.)2142 4129 y(F)o(ollo)n(wing)k(the)h(dimensions,)g(a)h(list)h (of)e(named)f(attrib)n(utes)i(are)2042 4229 y(used)36 b(to)g(describe)g(the)g(properties)f(of)g(the)i(dataset)f(\(e.g.,)j (data)2042 4329 y(range,)25 b(purpose,)e(associated)i(applications)e (\).)38 b(These)24 b(are)h(called)2042 4428 y(global)20 b(attrib)n(utes)h(and)g(are)f(separate)h(from)f(attrib)n(utes)h (associated)2042 4528 y(with)g(indi)n(vidual)d(v)n(ariables.)2142 4628 y(The)27 b(basic)h(units)g(of)f(named)g(data)h(in)f(a)i(netCDF)f (dataset)g(are)2042 4727 y(v)n(ariables,)h(which)e(are)g (multidimensional)f(arrays.)46 b(The)28 b(header)2042 4827 y(part)d(describes)f(each)h(v)n(ariable)f(by)h(its)h(name,)f (shape,)h(named)d(at-)2042 4926 y(trib)n(utes,)18 b(data)g(type,)g (array)f(size,)i(and)f(data)g(of)n(fset,)f(while)i(the)f(data)2042 5026 y(part)h(stores)h(the)f(array)f(v)n(alues)h(for)g(one)g(v)n (ariable)f(after)h(another)m(,)e(in)2042 5126 y(their)j(de\002ned)f (order)-5 b(.)2142 5225 y(T)e(o)25 b(support)f(v)n(ariable-size)g (arrays)g(\(e.g.,)h(data)g(gro)n(wing)e(with)2042 5325 y(time)g(stamps\),)f(netCDF)h(introduces)e(record)g(v)n(ariables)g(and) h(uses)1928 5574 y(2)p eop %%Page: 3 3 3 2 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.3) cvn H.B /DEST pdfmark end -114 0 a 45 1847 a @beginspecial 0 @llx 0 @lly 193 @urx 216 @ury 1980 @rwi @setspecial %%BeginDocument: fileformat.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def /col32 {0.776 0.784 0.784 srgb} bind def /col33 {0.776 0.784 0.784 srgb} bind def end save -6.0 222.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /reencdict 12 dict def /ReEncode { reencdict begin /newcodesandnames exch def /newfontname exch def /basefontname exch def /basefontdict basefontname findfont def /newfont basefontdict maxlength dict def basefontdict { exch dup /FID ne { dup /Encoding eq { exch dup length array copy newfont 3 1 roll put } { exch newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName newfontname put newcodesandnames aload pop 128 1 255 { newfont /Encoding get exch /.notdef put } for newcodesandnames length 2 idiv { newfont /Encoding get 3 1 roll put } repeat newfontname newfont definefont pop end } def /isovec [ 8#200 /grave 8#201 /acute 8#202 /circumflex 8#203 /tilde 8#204 /macron 8#205 /breve 8#206 /dotaccent 8#207 /dieresis 8#210 /ring 8#211 /cedilla 8#212 /hungarumlaut 8#213 /ogonek 8#214 /caron 8#220 /dotlessi 8#230 /oe 8#231 /OE 8#240 /space 8#241 /exclamdown 8#242 /cent 8#243 /sterling 8#244 /currency 8#245 /yen 8#246 /brokenbar 8#247 /section 8#250 /dieresis 8#251 /copyright 8#252 /ordfeminine 8#253 /guillemotleft 8#254 /logicalnot 8#255 /endash 8#256 /registered 8#257 /macron 8#260 /degree 8#261 /plusminus 8#262 /twosuperior 8#263 /threesuperior 8#264 /acute 8#265 /mu 8#266 /paragraph 8#267 /periodcentered 8#270 /cedilla 8#271 /onesuperior 8#272 /ordmasculine 8#273 /guillemotright 8#274 /onequarter 8#275 /onehalf 8#276 /threequarters 8#277 /questiondown 8#300 /Agrave 8#301 /Aacute 8#302 /Acircumflex 8#303 /Atilde 8#304 /Adieresis 8#305 /Aring 8#306 /AE 8#307 /Ccedilla 8#310 /Egrave 8#311 /Eacute 8#312 /Ecircumflex 8#313 /Edieresis 8#314 /Igrave 8#315 /Iacute 8#316 /Icircumflex 8#317 /Idieresis 8#320 /Eth 8#321 /Ntilde 8#322 /Ograve 8#323 /Oacute 8#324 /Ocircumflex 8#325 /Otilde 8#326 /Odieresis 8#327 /multiply 8#330 /Oslash 8#331 /Ugrave 8#332 /Uacute 8#333 /Ucircumflex 8#334 /Udieresis 8#335 /Yacute 8#336 /Thorn 8#337 /germandbls 8#340 /agrave 8#341 /aacute 8#342 /acircumflex 8#343 /atilde 8#344 /adieresis 8#345 /aring 8#346 /ae 8#347 /ccedilla 8#350 /egrave 8#351 /eacute 8#352 /ecircumflex 8#353 /edieresis 8#354 /igrave 8#355 /iacute 8#356 /icircumflex 8#357 /idieresis 8#360 /eth 8#361 /ntilde 8#362 /ograve 8#363 /oacute 8#364 /ocircumflex 8#365 /otilde 8#366 /odieresis 8#367 /divide 8#370 /oslash 8#371 /ugrave 8#372 /uacute 8#373 /ucircumflex 8#374 /udieresis 8#375 /yacute 8#376 /thorn 8#377 /ydieresis] def /Times-Roman /Times-Roman-iso isovec ReEncode /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 4697 m -1000 -1000 l 4311 -1000 l 4311 4697 l cp clip 0.06000 0.06000 sc % Polyline 7.500 slw n 600 3240 m 600 1590 l 2850 1590 l 2850 3240 l gs col30 1.00 shd ef gr gs col4 s gr % Polyline n 600 367 m 2850 367 l 2850 1567 l 600 1567 l cp gs col6 1.00 shd ef gr gs col26 s gr % Polyline n 600 150 m 2850 150 l 2850 345 l 600 345 l cp gs col11 1.00 shd ef gr gs col10 s gr % Polyline n 1425 3300 m 1725 3450 l 2025 3300 l gs col4 s gr % Polyline n 457 1597 m 382 1672 l 382 2347 l 307 2422 l 382 2497 l 382 3172 l 457 3247 l gs col4 s gr % Polyline n 600 2190 m 2850 2190 l gs col27 s gr /Times-Roman-iso ff 120.00 scf sf 2199 2587 m gs 1 -1 sc (nd) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 1920 2579 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 1695 1815 m gs 1 -1 sc (nd) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 1680 2273 m gs 1 -1 sc (th) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 1560 3645 m gs 1 -1 sc (st) col0 sh gr % Polyline n 1575 3150 m 1575 3375 l gs col4 s gr /Times-Roman-iso ff 105.00 scf sf 1792 3638 m gs 1 -1 sc (nd) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 2258 3660 m gs 1 -1 sc (th) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 1688 1665 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 262 3149 m gs 1 -1 sc 90.0 rot (variable-size arrays) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 255 1561 m gs 1 -1 sc 90.0 rot (fixed-size arrays) col0 sh gr % Polyline n 457 375 m 382 450 l 382 900 l 307 975 l 382 1050 l 382 1500 l 457 1575 l gs col26 s gr % Polyline 15.000 slw n 570 3255 m 570 122 l 2880 122 l 2880 3255 l gs col0 s gr /Times-Roman-iso ff 150.00 scf sf 404 3546 m gs 1 -1 sc (Interleaved records grow in the UNLIMITED) col0 sh gr % Polyline 7.500 slw n 600 592 m 2850 592 l gs col26 s gr % Polyline n 600 817 m 2850 817 l gs col26 s gr % Polyline n 600 1342 m 2850 1342 l gs col26 s gr % Polyline [60] 0 sd n 585 3247 m 2880 3247 l gs col4 s gr [] 0 sd /Times-Roman-iso ff 180.00 scf sf 728 2841 m gs 1 -1 sc (r record variables in order) col0 sh gr % Polyline n 600 1740 m 2850 1740 l gs col27 s gr % Polyline n 600 1890 m 2850 1890 l gs col27 s gr % Polyline n 600 2340 m 2850 2340 l gs col4 s gr % Polyline n 600 3090 m 2850 3090 l gs col4 s gr /Times-Roman-iso ff 180.00 scf sf 878 1514 m gs 1 -1 sc (n non-record variable) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 967 682 m gs 1 -1 sc (nd) col0 sh gr % Polyline n 1875 3150 m 1875 3375 l gs col4 s gr /Times-Roman-iso ff 150.00 scf sf 809 1718 m gs 1 -1 sc (1 record for 1 record variable) col0 sh gr /Times-Roman-iso ff 150.00 scf sf 608 3697 m gs 1 -1 sc (dimension for 1 , 2 , ... , r variables) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 862 2572 m gs 1 -1 sc (nd) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 974 1463 m gs 1 -1 sc (th) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 1148 307 m gs 1 -1 sc (netCDF Header) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 878 764 m gs 1 -1 sc (2 non-record variable) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 885 1665 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 878 547 m gs 1 -1 sc (1 non-record variable) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 967 457 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 150.00 scf sf 809 1868 m gs 1 -1 sc (1 record for 2 record variable) col0 sh gr /Times-Roman-iso ff 150.00 scf sf 809 2318 m gs 1 -1 sc (1 record for r record variable) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 885 2258 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 105.00 scf sf 885 1808 m gs 1 -1 sc (st) col0 sh gr /Times-Roman-iso ff 180.00 scf sf 772 2663 m gs 1 -1 sc (2 records for 1 , 2 , ... ,) col0 sh gr /Times-Roman-iso ff 120.00 scf sf 809 2790 m gs 1 -1 sc (th) col0 sh gr % Polyline 0.000 slw n 2755 2004 m 2755 2005 l 2756 2006 l 2757 2006 l 2758 2007 l 2759 2007 l 2760 2008 l 2761 2009 l 2762 2009 l 2763 2009 l 2764 2010 l 2764 2011 l 2765 2012 l 2766 2012 l 2767 2013 l 2767 2014 l 2768 2015 l 2769 2015 l 2770 2015 l 2771 2015 l 2772 2015 l 2773 2016 l 2774 2016 l 2775 2017 l 2776 2018 l 2777 2018 l 2778 2018 l 2779 2019 l 2779 2020 l 2780 2021 l 2781 2021 l 2782 2021 l 2783 2021 l 2784 2021 l 2785 2021 l 2786 2021 l 2787 2022 l 2788 2022 l 2788 2023 l 2789 2024 l 2790 2024 l 2791 2024 l 2792 2024 l 2793 2024 l 2794 2025 l 2795 2025 l 2796 2026 l 2797 2027 l 2797 2028 l 2797 2029 l 2798 2030 l 2799 2030 l 2800 2030 l 2801 2030 l 2802 2030 l 2803 2030 l 2804 2030 l 2805 2030 l 2806 2030 l 2807 2030 l 2808 2030 l 2809 2030 l 2810 2030 l 2811 2030 l 2812 2030 l 2813 2030 l 2814 2030 l 2815 2030 l 2816 2030 l 2817 2030 l 2818 2030 l 2818 2029 l 2818 2028 l 2819 2028 l 2819 2027 l 2820 2027 l 2821 2027 l 2822 2027 l 2823 2027 l 2824 2027 l 2825 2027 l 2826 2027 l 2827 2027 l 2827 2026 l 2827 2025 l 2828 2025 l 2828 2024 l 2829 2024 l 2830 2024 l 2830 2023 l 2830 2022 l 2831 2022 l 2831 2021 l 2832 2021 l 2833 2021 l 2834 2021 l 2835 2021 l 2836 2021 l 2837 2021 l 2838 2021 l 2839 2021 l 2840 2020 l 2841 2019 l 2842 2019 l 2843 2018 l 2844 2018 l 2845 2017 l 2846 2017 l 2846 2016 l 2847 2015 l 2849 2015 l 2850 2014 l 2851 2013 l 2852 2012 l 2853 2012 l 2854 2012 l 2854 2011 l 2854 2010 l 2855 2010 l 2855 2009 l 2856 2009 l 2857 2009 l 2858 2009 l 2859 2009 l 2860 2009 l 2861 2009 l 2862 2009 l 2863 2009 l 2863 2008 l 2863 2007 l 2864 2007 l 2864 2006 l 2865 2006 l 2866 2006 l 2867 2005 l 2868 2004 l 2869 2003 l 2869 2002 l 2869 2001 l 2870 2001 l 2870 2000 l 2871 2000 l 2872 2000 l 2873 2000 l 2874 2000 l 2875 2000 l 2876 2000 l 2877 2000 l 2878 2000 l 2878 1999 l 2878 1998 l 2879 1998 l 2879 1997 l 2880 1997 l 2881 1997 l 2882 1997 l 2883 1997 l 2884 1997 l 2885 1997 l 2886 1997 l 2886 1996 l 2887 1996 l 2887 1995 l 2888 1995 l 2888 1994 l 2889 1994 l 2890 1994 l 2891 1994 l 2892 1994 l 2893 1994 l 2894 1994 l 2895 1994 l 2896 1994 l 2897 1994 l 2898 1994 l 2899 1994 l 2899 1993 l 2899 1992 l 2900 1992 l 2900 1991 l 2901 1991 l 2902 1991 l 2903 1991 l 2904 1991 l 2905 1991 l 2906 1991 l 2906 1990 l 2907 1990 l 2909 1989 l 2910 1989 l 2911 1989 l 2911 1988 l 2912 1988 l 2913 1988 l 2914 1988 l 2914 1989 l 2915 1989 l 2916 1990 l 2917 1991 l 2918 1991 l 2919 1991 l 2920 1991 l 2921 1991 l 2922 1991 l 2923 1991 l 2924 1991 l 2925 1991 l 2926 1991 l 2927 1991 l 2928 1991 l 2930 1991 l 2931 1991 l 2932 1991 l 2933 1991 l 2934 1991 l 2935 1991 l 2936 1991 l 2937 1991 l 2938 1992 l 2939 1992 l 2940 1993 l 2941 1994 l 2942 1994 l 2943 1994 l 2944 1995 l 2944 1996 l 2945 1997 l 2946 1997 l 2947 1998 l 2948 1998 l 2949 1999 l 2950 2000 l 2951 2000 l 2952 2000 l 2953 2001 l 2953 2002 l 2954 2003 l 2955 2003 l 2956 2004 l 2956 2005 l 2957 2006 l 2958 2006 l 2959 2007 l 2959 2008 l 2959 2009 l 2959 2010 l 2959 2011 l 2959 2012 l 2959 2013 l 2959 2014 l 2959 2015 l 2959 2016 l 2959 2017 l 2960 2018 l 2961 2018 l 2962 2019 l 2962 2020 l 2962 2021 l 2962 2022 l 2962 2023 l 2962 2024 l 2962 2025 l 2962 2026 l 2962 2027 l 2962 2028 l 2962 2029 l 2962 2030 l 2962 2031 l 2962 2032 l 2962 2033 l 2962 2034 l 2962 2035 l 2962 2036 l 2962 2037 l 2962 2038 l 2963 2039 l 2964 2039 l 2965 2040 l 2965 2041 l 2965 2042 l 2965 2043 l 2965 2044 l 2965 2045 l 2965 2046 l 2965 2047 l 2965 2048 l 2965 2049 l 2965 2050 l 2965 2051 l 2965 2052 l 2965 2053 l 2965 2054 l 2965 2055 l 2965 2056 l 2965 2057 l 2965 2058 l 2965 2059 l 2965 2060 l 2965 2061 l 2965 2062 l 2965 2063 l 2965 2064 l 2965 2065 l 2965 2066 l 2965 2067 l 2965 2068 l 2965 2069 l 2964 2069 l 2963 2069 l 2962 2069 l 2961 2069 l 2960 2069 l 2959 2069 l 2958 2069 l 2957 2069 l 2956 2068 l 2956 2067 l 2955 2066 l 2954 2066 l 2953 2065 l 2953 2064 l 2952 2063 l 2951 2063 l 2950 2063 l 2949 2063 l 2948 2063 l 2947 2062 l 2947 2061 l 2946 2060 l 2945 2060 l 2944 2060 l 2943 2059 l 2942 2058 l 2941 2058 l 2940 2057 l 2939 2057 l 2938 2057 l 2937 2057 l 2936 2057 l 2935 2057 l 2934 2057 l 2933 2057 l 2932 2057 l 2931 2057 l 2930 2057 l 2929 2057 l 2928 2057 l 2927 2057 l 2926 2057 l 2925 2057 l 2924 2057 l 2923 2057 l 2922 2057 l 2921 2057 l 2920 2056 l 2920 2055 l 2919 2054 l 2918 2054 l 2917 2054 l 2916 2054 l 2915 2054 l 2914 2054 l 2913 2054 l 2912 2054 l 2911 2054 l 2910 2054 l 2909 2054 l 2908 2054 l 2907 2054 l 2906 2054 l 2905 2054 l 2904 2054 l 2903 2054 l 2902 2054 l 2901 2054 l 2900 2054 l 2900 2055 l 2899 2055 l 2899 2056 l 2899 2057 l 2898 2057 l 2897 2057 l 2896 2057 l 2895 2057 l 2894 2057 l 2893 2057 l 2892 2057 l 2891 2057 l 2890 2057 l 2889 2057 l 2888 2057 l 2887 2057 l 2886 2057 l 2885 2057 l 2885 2058 l 2884 2058 l 2884 2059 l 2884 2060 l 2883 2060 l 2882 2060 l 2882 2061 l 2881 2061 l 2881 2062 l 2881 2063 l 2880 2063 l 2879 2063 l 2878 2063 l 2877 2063 l 2876 2063 l 2875 2063 l 2874 2063 l 2873 2063 l 2873 2064 l 2872 2064 l 2872 2065 l 2872 2066 l 2871 2066 l 2870 2066 l 2870 2067 l 2869 2067 l 2869 2068 l 2869 2069 l 2868 2069 l 2867 2069 l 2866 2069 l 2865 2069 l 2864 2069 l 2863 2069 l 2862 2069 l 2861 2069 l 2860 2069 l 2859 2069 l 2858 2069 l 2857 2069 l 2856 2069 l 2855 2069 l 2855 2070 l 2854 2070 l 2854 2071 l 2854 2072 l 2853 2072 l 2852 2072 l 2851 2072 l 2850 2072 l 2849 2072 l 2848 2073 l 2847 2073 l 2846 2074 l 2845 2075 l 2844 2075 l 2843 2075 l 2843 2076 l 2842 2076 l 2842 2077 l 2842 2078 l 2841 2078 l 2840 2078 l 2839 2078 l 2838 2078 l 2837 2078 l 2837 2079 l 2836 2079 l 2836 2080 l 2836 2081 l 2835 2081 l 2834 2081 l 2833 2081 l 2832 2081 l 2831 2081 l 2830 2082 l 2829 2082 l 2828 2083 l 2827 2084 l 2826 2084 l 2825 2084 l 2824 2084 l 2823 2084 l 2822 2084 l 2821 2084 l 2820 2084 l 2819 2084 l 2819 2085 l 2818 2085 l 2818 2086 l 2818 2087 l 2817 2087 l 2816 2087 l 2815 2087 l 2814 2087 l 2813 2087 l 2812 2087 l 2811 2087 l 2810 2087 l 2809 2087 l 2808 2087 l 2807 2087 l 2806 2087 l 2805 2087 l 2804 2087 l 2803 2087 l 2802 2087 l 2801 2087 l 2800 2087 l 2799 2087 l 2798 2087 l 2797 2087 l 2796 2087 l 2795 2087 l 2794 2087 l 2793 2087 l 2792 2087 l 2791 2087 l 2790 2087 l 2789 2087 l 2788 2087 l 2787 2087 l 2786 2087 l 2785 2086 l 2785 2085 l 2784 2084 l 2783 2084 l 2782 2083 l 2782 2082 l 2781 2081 l 2780 2081 l 2779 2080 l 2779 2079 l 2778 2078 l 2777 2078 l 2776 2077 l 2776 2076 l 2775 2075 l 2774 2075 l 2773 2074 l 2773 2073 l 2772 2072 l 2771 2072 l 2770 2072 l 2769 2072 l 2768 2072 l 2767 2071 l 2767 2070 l 2766 2069 l 2765 2069 l 2764 2069 l 2763 2069 l 2762 2069 l 2761 2069 l 2760 2069 l 2759 2069 l 2758 2068 l 2758 2067 l 2757 2066 l 2756 2066 l 2755 2066 l 2754 2066 l 2753 2066 l 2752 2065 l 2752 2064 l 2752 2063 l 2752 2062 l 2752 2061 l 2752 2060 l 2752 2059 l 2752 2058 l 2752 2057 l 2752 2056 l 2752 2055 l 2752 2054 l 2752 2053 l 2752 2052 l 2752 2051 l 2752 2050 l 2752 2049 l 2753 2049 l 2753 2048 l 2754 2048 l 2755 2048 l 2755 2047 l 2755 2046 l 2755 2045 l 2755 2044 l 2755 2043 l 2755 2042 l 2755 2041 l 2755 2040 l 2755 2039 l 2755 2038 l 2755 2037 l 2755 2036 l 2755 2035 l 2755 2034 l 2754 2033 l 2753 2033 l 2752 2032 l 2752 2031 l 2752 2030 l 2752 2029 l 2752 2028 l 2752 2027 l 2752 2026 l 2752 2025 l 2752 2024 l 2752 2023 l 2752 2022 l 2752 2021 l 2752 2020 l 2752 2019 l 2752 2018 l 2752 2017 l 2752 2016 l 2752 2015 l 2752 2014 l 2752 2013 l 2752 2012 l 2752 2011 l 2752 2010 l 2752 2009 l 2752 2008 l 2752 2007 l 2752 2006 l 2752 2005 l 2752 2004 l 2752 2003 l 2753 2003 l 2754 2003 l cp gs col7 1.00 shd ef gr % Polyline n 483 1037 m 483 1038 l 484 1039 l 485 1039 l 486 1040 l 487 1040 l 488 1041 l 489 1042 l 490 1042 l 491 1042 l 492 1043 l 492 1044 l 493 1045 l 494 1045 l 495 1046 l 495 1047 l 496 1048 l 497 1048 l 498 1048 l 499 1048 l 500 1048 l 501 1049 l 502 1049 l 503 1050 l 504 1051 l 505 1051 l 506 1051 l 507 1052 l 507 1053 l 508 1054 l 509 1054 l 510 1054 l 511 1054 l 512 1054 l 513 1054 l 514 1054 l 515 1055 l 516 1055 l 516 1056 l 517 1057 l 518 1057 l 519 1057 l 520 1057 l 521 1057 l 522 1058 l 523 1058 l 524 1059 l 525 1060 l 525 1061 l 525 1062 l 526 1063 l 527 1063 l 528 1063 l 529 1063 l 530 1063 l 531 1063 l 532 1063 l 533 1063 l 534 1063 l 535 1063 l 536 1063 l 537 1063 l 538 1063 l 539 1063 l 540 1063 l 541 1063 l 542 1063 l 543 1063 l 544 1063 l 545 1063 l 546 1063 l 546 1062 l 546 1061 l 547 1061 l 547 1060 l 548 1060 l 549 1060 l 550 1060 l 551 1060 l 552 1060 l 553 1060 l 554 1060 l 555 1060 l 555 1059 l 555 1058 l 556 1058 l 556 1057 l 557 1057 l 558 1057 l 558 1056 l 558 1055 l 559 1055 l 559 1054 l 560 1054 l 561 1054 l 562 1054 l 563 1054 l 564 1054 l 565 1054 l 566 1054 l 567 1054 l 568 1053 l 569 1052 l 570 1052 l 571 1051 l 572 1051 l 573 1050 l 574 1050 l 574 1049 l 575 1048 l 577 1048 l 578 1047 l 579 1046 l 580 1045 l 581 1045 l 582 1045 l 582 1044 l 582 1043 l 583 1043 l 583 1042 l 584 1042 l 585 1042 l 586 1042 l 587 1042 l 588 1042 l 589 1042 l 590 1042 l 591 1042 l 591 1041 l 591 1040 l 592 1040 l 592 1039 l 593 1039 l 594 1039 l 595 1038 l 596 1037 l 597 1036 l 597 1035 l 597 1034 l 598 1034 l 598 1033 l 599 1033 l 600 1033 l 601 1033 l 602 1033 l 603 1033 l 604 1033 l 605 1033 l 606 1033 l 606 1032 l 606 1031 l 607 1031 l 607 1030 l 608 1030 l 609 1030 l 610 1030 l 611 1030 l 612 1030 l 613 1030 l 614 1030 l 614 1029 l 615 1029 l 615 1028 l 616 1028 l 616 1027 l 617 1027 l 618 1027 l 619 1027 l 620 1027 l 621 1027 l 622 1027 l 623 1027 l 624 1027 l 625 1027 l 626 1027 l 627 1027 l 627 1026 l 627 1025 l 628 1025 l 628 1024 l 629 1024 l 630 1024 l 631 1024 l 632 1024 l 633 1024 l 634 1024 l 634 1023 l 635 1023 l 637 1022 l 638 1022 l 639 1022 l 639 1021 l 640 1021 l 641 1021 l 642 1021 l 642 1022 l 643 1022 l 644 1023 l 645 1024 l 646 1024 l 647 1024 l 648 1024 l 649 1024 l 650 1024 l 651 1024 l 652 1024 l 653 1024 l 654 1024 l 655 1024 l 656 1024 l 658 1024 l 659 1024 l 660 1024 l 661 1024 l 662 1024 l 663 1024 l 664 1024 l 665 1024 l 666 1025 l 667 1025 l 668 1026 l 669 1027 l 670 1027 l 671 1027 l 672 1028 l 672 1029 l 673 1030 l 674 1030 l 675 1031 l 676 1031 l 677 1032 l 678 1033 l 679 1033 l 680 1033 l 681 1034 l 681 1035 l 682 1036 l 683 1036 l 684 1037 l 684 1038 l 685 1039 l 686 1039 l 687 1040 l 687 1041 l 687 1042 l 687 1043 l 687 1044 l 687 1045 l 687 1046 l 687 1047 l 687 1048 l 687 1049 l 687 1050 l 688 1051 l 689 1051 l 690 1052 l 690 1053 l 690 1054 l 690 1055 l 690 1056 l 690 1057 l 690 1058 l 690 1059 l 690 1060 l 690 1061 l 690 1062 l 690 1063 l 690 1064 l 690 1065 l 690 1066 l 690 1067 l 690 1068 l 690 1069 l 690 1070 l 690 1071 l 691 1072 l 692 1072 l 693 1073 l 693 1074 l 693 1075 l 693 1076 l 693 1077 l 693 1078 l 693 1079 l 693 1080 l 693 1081 l 693 1082 l 693 1083 l 693 1084 l 693 1085 l 693 1086 l 693 1087 l 693 1088 l 693 1089 l 693 1090 l 693 1091 l 693 1092 l 693 1093 l 693 1094 l 693 1095 l 693 1096 l 693 1097 l 693 1098 l 693 1099 l 693 1100 l 693 1101 l 693 1102 l 692 1102 l 691 1102 l 690 1102 l 689 1102 l 688 1102 l 687 1102 l 686 1102 l 685 1102 l 684 1101 l 684 1100 l 683 1099 l 682 1099 l 681 1098 l 681 1097 l 680 1096 l 679 1096 l 678 1096 l 677 1096 l 676 1096 l 675 1095 l 675 1094 l 674 1093 l 673 1093 l 672 1093 l 671 1092 l 670 1091 l 669 1091 l 668 1090 l 667 1090 l 666 1090 l 665 1090 l 664 1090 l 663 1090 l 662 1090 l 661 1090 l 660 1090 l 659 1090 l 658 1090 l 657 1090 l 656 1090 l 655 1090 l 654 1090 l 653 1090 l 652 1090 l 651 1090 l 650 1090 l 649 1090 l 648 1089 l 648 1088 l 647 1087 l 646 1087 l 645 1087 l 644 1087 l 643 1087 l 642 1087 l 641 1087 l 640 1087 l 639 1087 l 638 1087 l 637 1087 l 636 1087 l 635 1087 l 634 1087 l 633 1087 l 632 1087 l 631 1087 l 630 1087 l 629 1087 l 628 1087 l 628 1088 l 627 1088 l 627 1089 l 627 1090 l 626 1090 l 625 1090 l 624 1090 l 623 1090 l 622 1090 l 621 1090 l 620 1090 l 619 1090 l 618 1090 l 617 1090 l 616 1090 l 615 1090 l 614 1090 l 613 1090 l 613 1091 l 612 1091 l 612 1092 l 612 1093 l 611 1093 l 610 1093 l 610 1094 l 609 1094 l 609 1095 l 609 1096 l 608 1096 l 607 1096 l 606 1096 l 605 1096 l 604 1096 l 603 1096 l 602 1096 l 601 1096 l 601 1097 l 600 1097 l 600 1098 l 600 1099 l 599 1099 l 598 1099 l 598 1100 l 597 1100 l 597 1101 l 597 1102 l 596 1102 l 595 1102 l 594 1102 l 593 1102 l 592 1102 l 591 1102 l 590 1102 l 589 1102 l 588 1102 l 587 1102 l 586 1102 l 585 1102 l 584 1102 l 583 1102 l 583 1103 l 582 1103 l 582 1104 l 582 1105 l 581 1105 l 580 1105 l 579 1105 l 578 1105 l 577 1105 l 576 1106 l 575 1106 l 574 1107 l 573 1108 l 572 1108 l 571 1108 l 571 1109 l 570 1109 l 570 1110 l 570 1111 l 569 1111 l 568 1111 l 567 1111 l 566 1111 l 565 1111 l 565 1112 l 564 1112 l 564 1113 l 564 1114 l 563 1114 l 562 1114 l 561 1114 l 560 1114 l 559 1114 l 558 1115 l 557 1115 l 556 1116 l 555 1117 l 554 1117 l 553 1117 l 552 1117 l 551 1117 l 550 1117 l 549 1117 l 548 1117 l 547 1117 l 547 1118 l 546 1118 l 546 1119 l 546 1120 l 545 1120 l 544 1120 l 543 1120 l 542 1120 l 541 1120 l 540 1120 l 539 1120 l 538 1120 l 537 1120 l 536 1120 l 535 1120 l 534 1120 l 533 1120 l 532 1120 l 531 1120 l 530 1120 l 529 1120 l 528 1120 l 527 1120 l 526 1120 l 525 1120 l 524 1120 l 523 1120 l 522 1120 l 521 1120 l 520 1120 l 519 1120 l 518 1120 l 517 1120 l 516 1120 l 515 1120 l 514 1120 l 513 1119 l 513 1118 l 512 1117 l 511 1117 l 510 1116 l 510 1115 l 509 1114 l 508 1114 l 507 1113 l 507 1112 l 506 1111 l 505 1111 l 504 1110 l 504 1109 l 503 1108 l 502 1108 l 501 1107 l 501 1106 l 500 1105 l 499 1105 l 498 1105 l 497 1105 l 496 1105 l 495 1104 l 495 1103 l 494 1102 l 493 1102 l 492 1102 l 491 1102 l 490 1102 l 489 1102 l 488 1102 l 487 1102 l 486 1101 l 486 1100 l 485 1099 l 484 1099 l 483 1099 l 482 1099 l 481 1099 l 480 1098 l 480 1097 l 480 1096 l 480 1095 l 480 1094 l 480 1093 l 480 1092 l 480 1091 l 480 1090 l 480 1089 l 480 1088 l 480 1087 l 480 1086 l 480 1085 l 480 1084 l 480 1083 l 480 1082 l 481 1082 l 481 1081 l 482 1081 l 483 1081 l 483 1080 l 483 1079 l 483 1078 l 483 1077 l 483 1076 l 483 1075 l 483 1074 l 483 1073 l 483 1072 l 483 1071 l 483 1070 l 483 1069 l 483 1068 l 483 1067 l 482 1066 l 481 1066 l 480 1065 l 480 1064 l 480 1063 l 480 1062 l 480 1061 l 480 1060 l 480 1059 l 480 1058 l 480 1057 l 480 1056 l 480 1055 l 480 1054 l 480 1053 l 480 1052 l 480 1051 l 480 1050 l 480 1049 l 480 1048 l 480 1047 l 480 1046 l 480 1045 l 480 1044 l 480 1043 l 480 1042 l 480 1041 l 480 1040 l 480 1039 l 480 1038 l 480 1037 l 480 1036 l 481 1036 l 482 1036 l cp gs col7 1.00 shd ef gr % Polyline n 483 2004 m 483 2005 l 484 2006 l 485 2006 l 486 2007 l 487 2007 l 488 2008 l 489 2009 l 490 2009 l 491 2009 l 492 2010 l 492 2011 l 493 2012 l 494 2012 l 495 2013 l 495 2014 l 496 2015 l 497 2015 l 498 2015 l 499 2015 l 500 2015 l 501 2016 l 502 2016 l 503 2017 l 504 2018 l 505 2018 l 506 2018 l 507 2019 l 507 2020 l 508 2021 l 509 2021 l 510 2021 l 511 2021 l 512 2021 l 513 2021 l 514 2021 l 515 2022 l 516 2022 l 516 2023 l 517 2024 l 518 2024 l 519 2024 l 520 2024 l 521 2024 l 522 2025 l 523 2025 l 524 2026 l 525 2027 l 525 2028 l 525 2029 l 526 2030 l 527 2030 l 528 2030 l 529 2030 l 530 2030 l 531 2030 l 532 2030 l 533 2030 l 534 2030 l 535 2030 l 536 2030 l 537 2030 l 538 2030 l 539 2030 l 540 2030 l 541 2030 l 542 2030 l 543 2030 l 544 2030 l 545 2030 l 546 2030 l 546 2029 l 546 2028 l 547 2028 l 547 2027 l 548 2027 l 549 2027 l 550 2027 l 551 2027 l 552 2027 l 553 2027 l 554 2027 l 555 2027 l 555 2026 l 555 2025 l 556 2025 l 556 2024 l 557 2024 l 558 2024 l 558 2023 l 558 2022 l 559 2022 l 559 2021 l 560 2021 l 561 2021 l 562 2021 l 563 2021 l 564 2021 l 565 2021 l 566 2021 l 567 2021 l 568 2020 l 569 2019 l 570 2019 l 571 2018 l 572 2018 l 573 2017 l 574 2017 l 574 2016 l 575 2015 l 577 2015 l 578 2014 l 579 2013 l 580 2012 l 581 2012 l 582 2012 l 582 2011 l 582 2010 l 583 2010 l 583 2009 l 584 2009 l 585 2009 l 586 2009 l 587 2009 l 588 2009 l 589 2009 l 590 2009 l 591 2009 l 591 2008 l 591 2007 l 592 2007 l 592 2006 l 593 2006 l 594 2006 l 595 2005 l 596 2004 l 597 2003 l 597 2002 l 597 2001 l 598 2001 l 598 2000 l 599 2000 l 600 2000 l 601 2000 l 602 2000 l 603 2000 l 604 2000 l 605 2000 l 606 2000 l 606 1999 l 606 1998 l 607 1998 l 607 1997 l 608 1997 l 609 1997 l 610 1997 l 611 1997 l 612 1997 l 613 1997 l 614 1997 l 614 1996 l 615 1996 l 615 1995 l 616 1995 l 616 1994 l 617 1994 l 618 1994 l 619 1994 l 620 1994 l 621 1994 l 622 1994 l 623 1994 l 624 1994 l 625 1994 l 626 1994 l 627 1994 l 627 1993 l 627 1992 l 628 1992 l 628 1991 l 629 1991 l 630 1991 l 631 1991 l 632 1991 l 633 1991 l 634 1991 l 634 1990 l 635 1990 l 637 1989 l 638 1989 l 639 1989 l 639 1988 l 640 1988 l 641 1988 l 642 1988 l 642 1989 l 643 1989 l 644 1990 l 645 1991 l 646 1991 l 647 1991 l 648 1991 l 649 1991 l 650 1991 l 651 1991 l 652 1991 l 653 1991 l 654 1991 l 655 1991 l 656 1991 l 658 1991 l 659 1991 l 660 1991 l 661 1991 l 662 1991 l 663 1991 l 664 1991 l 665 1991 l 666 1992 l 667 1992 l 668 1993 l 669 1994 l 670 1994 l 671 1994 l 672 1995 l 672 1996 l 673 1997 l 674 1997 l 675 1998 l 676 1998 l 677 1999 l 678 2000 l 679 2000 l 680 2000 l 681 2001 l 681 2002 l 682 2003 l 683 2003 l 684 2004 l 684 2005 l 685 2006 l 686 2006 l 687 2007 l 687 2008 l 687 2009 l 687 2010 l 687 2011 l 687 2012 l 687 2013 l 687 2014 l 687 2015 l 687 2016 l 687 2017 l 688 2018 l 689 2018 l 690 2019 l 690 2020 l 690 2021 l 690 2022 l 690 2023 l 690 2024 l 690 2025 l 690 2026 l 690 2027 l 690 2028 l 690 2029 l 690 2030 l 690 2031 l 690 2032 l 690 2033 l 690 2034 l 690 2035 l 690 2036 l 690 2037 l 690 2038 l 691 2039 l 692 2039 l 693 2040 l 693 2041 l 693 2042 l 693 2043 l 693 2044 l 693 2045 l 693 2046 l 693 2047 l 693 2048 l 693 2049 l 693 2050 l 693 2051 l 693 2052 l 693 2053 l 693 2054 l 693 2055 l 693 2056 l 693 2057 l 693 2058 l 693 2059 l 693 2060 l 693 2061 l 693 2062 l 693 2063 l 693 2064 l 693 2065 l 693 2066 l 693 2067 l 693 2068 l 693 2069 l 692 2069 l 691 2069 l 690 2069 l 689 2069 l 688 2069 l 687 2069 l 686 2069 l 685 2069 l 684 2068 l 684 2067 l 683 2066 l 682 2066 l 681 2065 l 681 2064 l 680 2063 l 679 2063 l 678 2063 l 677 2063 l 676 2063 l 675 2062 l 675 2061 l 674 2060 l 673 2060 l 672 2060 l 671 2059 l 670 2058 l 669 2058 l 668 2057 l 667 2057 l 666 2057 l 665 2057 l 664 2057 l 663 2057 l 662 2057 l 661 2057 l 660 2057 l 659 2057 l 658 2057 l 657 2057 l 656 2057 l 655 2057 l 654 2057 l 653 2057 l 652 2057 l 651 2057 l 650 2057 l 649 2057 l 648 2056 l 648 2055 l 647 2054 l 646 2054 l 645 2054 l 644 2054 l 643 2054 l 642 2054 l 641 2054 l 640 2054 l 639 2054 l 638 2054 l 637 2054 l 636 2054 l 635 2054 l 634 2054 l 633 2054 l 632 2054 l 631 2054 l 630 2054 l 629 2054 l 628 2054 l 628 2055 l 627 2055 l 627 2056 l 627 2057 l 626 2057 l 625 2057 l 624 2057 l 623 2057 l 622 2057 l 621 2057 l 620 2057 l 619 2057 l 618 2057 l 617 2057 l 616 2057 l 615 2057 l 614 2057 l 613 2057 l 613 2058 l 612 2058 l 612 2059 l 612 2060 l 611 2060 l 610 2060 l 610 2061 l 609 2061 l 609 2062 l 609 2063 l 608 2063 l 607 2063 l 606 2063 l 605 2063 l 604 2063 l 603 2063 l 602 2063 l 601 2063 l 601 2064 l 600 2064 l 600 2065 l 600 2066 l 599 2066 l 598 2066 l 598 2067 l 597 2067 l 597 2068 l 597 2069 l 596 2069 l 595 2069 l 594 2069 l 593 2069 l 592 2069 l 591 2069 l 590 2069 l 589 2069 l 588 2069 l 587 2069 l 586 2069 l 585 2069 l 584 2069 l 583 2069 l 583 2070 l 582 2070 l 582 2071 l 582 2072 l 581 2072 l 580 2072 l 579 2072 l 578 2072 l 577 2072 l 576 2073 l 575 2073 l 574 2074 l 573 2075 l 572 2075 l 571 2075 l 571 2076 l 570 2076 l 570 2077 l 570 2078 l 569 2078 l 568 2078 l 567 2078 l 566 2078 l 565 2078 l 565 2079 l 564 2079 l 564 2080 l 564 2081 l 563 2081 l 562 2081 l 561 2081 l 560 2081 l 559 2081 l 558 2082 l 557 2082 l 556 2083 l 555 2084 l 554 2084 l 553 2084 l 552 2084 l 551 2084 l 550 2084 l 549 2084 l 548 2084 l 547 2084 l 547 2085 l 546 2085 l 546 2086 l 546 2087 l 545 2087 l 544 2087 l 543 2087 l 542 2087 l 541 2087 l 540 2087 l 539 2087 l 538 2087 l 537 2087 l 536 2087 l 535 2087 l 534 2087 l 533 2087 l 532 2087 l 531 2087 l 530 2087 l 529 2087 l 528 2087 l 527 2087 l 526 2087 l 525 2087 l 524 2087 l 523 2087 l 522 2087 l 521 2087 l 520 2087 l 519 2087 l 518 2087 l 517 2087 l 516 2087 l 515 2087 l 514 2087 l 513 2086 l 513 2085 l 512 2084 l 511 2084 l 510 2083 l 510 2082 l 509 2081 l 508 2081 l 507 2080 l 507 2079 l 506 2078 l 505 2078 l 504 2077 l 504 2076 l 503 2075 l 502 2075 l 501 2074 l 501 2073 l 500 2072 l 499 2072 l 498 2072 l 497 2072 l 496 2072 l 495 2071 l 495 2070 l 494 2069 l 493 2069 l 492 2069 l 491 2069 l 490 2069 l 489 2069 l 488 2069 l 487 2069 l 486 2068 l 486 2067 l 485 2066 l 484 2066 l 483 2066 l 482 2066 l 481 2066 l 480 2065 l 480 2064 l 480 2063 l 480 2062 l 480 2061 l 480 2060 l 480 2059 l 480 2058 l 480 2057 l 480 2056 l 480 2055 l 480 2054 l 480 2053 l 480 2052 l 480 2051 l 480 2050 l 480 2049 l 481 2049 l 481 2048 l 482 2048 l 483 2048 l 483 2047 l 483 2046 l 483 2045 l 483 2044 l 483 2043 l 483 2042 l 483 2041 l 483 2040 l 483 2039 l 483 2038 l 483 2037 l 483 2036 l 483 2035 l 483 2034 l 482 2033 l 481 2033 l 480 2032 l 480 2031 l 480 2030 l 480 2029 l 480 2028 l 480 2027 l 480 2026 l 480 2025 l 480 2024 l 480 2023 l 480 2022 l 480 2021 l 480 2020 l 480 2019 l 480 2018 l 480 2017 l 480 2016 l 480 2015 l 480 2014 l 480 2013 l 480 2012 l 480 2011 l 480 2010 l 480 2009 l 480 2008 l 480 2007 l 480 2006 l 480 2005 l 480 2004 l 480 2003 l 481 2003 l 482 2003 l cp gs col7 1.00 shd ef gr % Polyline n 2755 3129 m 2755 3130 l 2756 3131 l 2757 3131 l 2758 3132 l 2759 3132 l 2760 3133 l 2761 3134 l 2762 3134 l 2763 3134 l 2764 3135 l 2764 3136 l 2765 3137 l 2766 3137 l 2767 3138 l 2767 3139 l 2768 3140 l 2769 3140 l 2770 3140 l 2771 3140 l 2772 3140 l 2773 3141 l 2774 3141 l 2775 3142 l 2776 3143 l 2777 3143 l 2778 3143 l 2779 3144 l 2779 3145 l 2780 3146 l 2781 3146 l 2782 3146 l 2783 3146 l 2784 3146 l 2785 3146 l 2786 3146 l 2787 3147 l 2788 3147 l 2788 3148 l 2789 3149 l 2790 3149 l 2791 3149 l 2792 3149 l 2793 3149 l 2794 3150 l 2795 3150 l 2796 3151 l 2797 3152 l 2797 3153 l 2797 3154 l 2798 3155 l 2799 3155 l 2800 3155 l 2801 3155 l 2802 3155 l 2803 3155 l 2804 3155 l 2805 3155 l 2806 3155 l 2807 3155 l 2808 3155 l 2809 3155 l 2810 3155 l 2811 3155 l 2812 3155 l 2813 3155 l 2814 3155 l 2815 3155 l 2816 3155 l 2817 3155 l 2818 3155 l 2818 3154 l 2818 3153 l 2819 3153 l 2819 3152 l 2820 3152 l 2821 3152 l 2822 3152 l 2823 3152 l 2824 3152 l 2825 3152 l 2826 3152 l 2827 3152 l 2827 3151 l 2827 3150 l 2828 3150 l 2828 3149 l 2829 3149 l 2830 3149 l 2830 3148 l 2830 3147 l 2831 3147 l 2831 3146 l 2832 3146 l 2833 3146 l 2834 3146 l 2835 3146 l 2836 3146 l 2837 3146 l 2838 3146 l 2839 3146 l 2840 3145 l 2841 3144 l 2842 3144 l 2843 3143 l 2844 3143 l 2845 3142 l 2846 3142 l 2846 3141 l 2847 3140 l 2849 3140 l 2850 3139 l 2851 3138 l 2852 3137 l 2853 3137 l 2854 3137 l 2854 3136 l 2854 3135 l 2855 3135 l 2855 3134 l 2856 3134 l 2857 3134 l 2858 3134 l 2859 3134 l 2860 3134 l 2861 3134 l 2862 3134 l 2863 3134 l 2863 3133 l 2863 3132 l 2864 3132 l 2864 3131 l 2865 3131 l 2866 3131 l 2867 3130 l 2868 3129 l 2869 3128 l 2869 3127 l 2869 3126 l 2870 3126 l 2870 3125 l 2871 3125 l 2872 3125 l 2873 3125 l 2874 3125 l 2875 3125 l 2876 3125 l 2877 3125 l 2878 3125 l 2878 3124 l 2878 3123 l 2879 3123 l 2879 3122 l 2880 3122 l 2881 3122 l 2882 3122 l 2883 3122 l 2884 3122 l 2885 3122 l 2886 3122 l 2886 3121 l 2887 3121 l 2887 3120 l 2888 3120 l 2888 3119 l 2889 3119 l 2890 3119 l 2891 3119 l 2892 3119 l 2893 3119 l 2894 3119 l 2895 3119 l 2896 3119 l 2897 3119 l 2898 3119 l 2899 3119 l 2899 3118 l 2899 3117 l 2900 3117 l 2900 3116 l 2901 3116 l 2902 3116 l 2903 3116 l 2904 3116 l 2905 3116 l 2906 3116 l 2906 3115 l 2907 3115 l 2909 3114 l 2910 3114 l 2911 3114 l 2911 3113 l 2912 3113 l 2913 3113 l 2914 3113 l 2914 3114 l 2915 3114 l 2916 3115 l 2917 3116 l 2918 3116 l 2919 3116 l 2920 3116 l 2921 3116 l 2922 3116 l 2923 3116 l 2924 3116 l 2925 3116 l 2926 3116 l 2927 3116 l 2928 3116 l 2930 3116 l 2931 3116 l 2932 3116 l 2933 3116 l 2934 3116 l 2935 3116 l 2936 3116 l 2937 3116 l 2938 3117 l 2939 3117 l 2940 3118 l 2941 3119 l 2942 3119 l 2943 3119 l 2944 3120 l 2944 3121 l 2945 3122 l 2946 3122 l 2947 3123 l 2948 3123 l 2949 3124 l 2950 3125 l 2951 3125 l 2952 3125 l 2953 3126 l 2953 3127 l 2954 3128 l 2955 3128 l 2956 3129 l 2956 3130 l 2957 3131 l 2958 3131 l 2959 3132 l 2959 3133 l 2959 3134 l 2959 3135 l 2959 3136 l 2959 3137 l 2959 3138 l 2959 3139 l 2959 3140 l 2959 3141 l 2959 3142 l 2960 3143 l 2961 3143 l 2962 3144 l 2962 3145 l 2962 3146 l 2962 3147 l 2962 3148 l 2962 3149 l 2962 3150 l 2962 3151 l 2962 3152 l 2962 3153 l 2962 3154 l 2962 3155 l 2962 3156 l 2962 3157 l 2962 3158 l 2962 3159 l 2962 3160 l 2962 3161 l 2962 3162 l 2962 3163 l 2963 3164 l 2964 3164 l 2965 3165 l 2965 3166 l 2965 3167 l 2965 3168 l 2965 3169 l 2965 3170 l 2965 3171 l 2965 3172 l 2965 3173 l 2965 3174 l 2965 3175 l 2965 3176 l 2965 3177 l 2965 3178 l 2965 3179 l 2965 3180 l 2965 3181 l 2965 3182 l 2965 3183 l 2965 3184 l 2965 3185 l 2965 3186 l 2965 3187 l 2965 3188 l 2965 3189 l 2965 3190 l 2965 3191 l 2965 3192 l 2965 3193 l 2965 3194 l 2964 3194 l 2963 3194 l 2962 3194 l 2961 3194 l 2960 3194 l 2959 3194 l 2958 3194 l 2957 3194 l 2956 3193 l 2956 3192 l 2955 3191 l 2954 3191 l 2953 3190 l 2953 3189 l 2952 3188 l 2951 3188 l 2950 3188 l 2949 3188 l 2948 3188 l 2947 3187 l 2947 3186 l 2946 3185 l 2945 3185 l 2944 3185 l 2943 3184 l 2942 3183 l 2941 3183 l 2940 3182 l 2939 3182 l 2938 3182 l 2937 3182 l 2936 3182 l 2935 3182 l 2934 3182 l 2933 3182 l 2932 3182 l 2931 3182 l 2930 3182 l 2929 3182 l 2928 3182 l 2927 3182 l 2926 3182 l 2925 3182 l 2924 3182 l 2923 3182 l 2922 3182 l 2921 3182 l 2920 3181 l 2920 3180 l 2919 3179 l 2918 3179 l 2917 3179 l 2916 3179 l 2915 3179 l 2914 3179 l 2913 3179 l 2912 3179 l 2911 3179 l 2910 3179 l 2909 3179 l 2908 3179 l 2907 3179 l 2906 3179 l 2905 3179 l 2904 3179 l 2903 3179 l 2902 3179 l 2901 3179 l 2900 3179 l 2900 3180 l 2899 3180 l 2899 3181 l 2899 3182 l 2898 3182 l 2897 3182 l 2896 3182 l 2895 3182 l 2894 3182 l 2893 3182 l 2892 3182 l 2891 3182 l 2890 3182 l 2889 3182 l 2888 3182 l 2887 3182 l 2886 3182 l 2885 3182 l 2885 3183 l 2884 3183 l 2884 3184 l 2884 3185 l 2883 3185 l 2882 3185 l 2882 3186 l 2881 3186 l 2881 3187 l 2881 3188 l 2880 3188 l 2879 3188 l 2878 3188 l 2877 3188 l 2876 3188 l 2875 3188 l 2874 3188 l 2873 3188 l 2873 3189 l 2872 3189 l 2872 3190 l 2872 3191 l 2871 3191 l 2870 3191 l 2870 3192 l 2869 3192 l 2869 3193 l 2869 3194 l 2868 3194 l 2867 3194 l 2866 3194 l 2865 3194 l 2864 3194 l 2863 3194 l 2862 3194 l 2861 3194 l 2860 3194 l 2859 3194 l 2858 3194 l 2857 3194 l 2856 3194 l 2855 3194 l 2855 3195 l 2854 3195 l 2854 3196 l 2854 3197 l 2853 3197 l 2852 3197 l 2851 3197 l 2850 3197 l 2849 3197 l 2848 3198 l 2847 3198 l 2846 3199 l 2845 3200 l 2844 3200 l 2843 3200 l 2843 3201 l 2842 3201 l 2842 3202 l 2842 3203 l 2841 3203 l 2840 3203 l 2839 3203 l 2838 3203 l 2837 3203 l 2837 3204 l 2836 3204 l 2836 3205 l 2836 3206 l 2835 3206 l 2834 3206 l 2833 3206 l 2832 3206 l 2831 3206 l 2830 3207 l 2829 3207 l 2828 3208 l 2827 3209 l 2826 3209 l 2825 3209 l 2824 3209 l 2823 3209 l 2822 3209 l 2821 3209 l 2820 3209 l 2819 3209 l 2819 3210 l 2818 3210 l 2818 3211 l 2818 3212 l 2817 3212 l 2816 3212 l 2815 3212 l 2814 3212 l 2813 3212 l 2812 3212 l 2811 3212 l 2810 3212 l 2809 3212 l 2808 3212 l 2807 3212 l 2806 3212 l 2805 3212 l 2804 3212 l 2803 3212 l 2802 3212 l 2801 3212 l 2800 3212 l 2799 3212 l 2798 3212 l 2797 3212 l 2796 3212 l 2795 3212 l 2794 3212 l 2793 3212 l 2792 3212 l 2791 3212 l 2790 3212 l 2789 3212 l 2788 3212 l 2787 3212 l 2786 3212 l 2785 3211 l 2785 3210 l 2784 3209 l 2783 3209 l 2782 3208 l 2782 3207 l 2781 3206 l 2780 3206 l 2779 3205 l 2779 3204 l 2778 3203 l 2777 3203 l 2776 3202 l 2776 3201 l 2775 3200 l 2774 3200 l 2773 3199 l 2773 3198 l 2772 3197 l 2771 3197 l 2770 3197 l 2769 3197 l 2768 3197 l 2767 3196 l 2767 3195 l 2766 3194 l 2765 3194 l 2764 3194 l 2763 3194 l 2762 3194 l 2761 3194 l 2760 3194 l 2759 3194 l 2758 3193 l 2758 3192 l 2757 3191 l 2756 3191 l 2755 3191 l 2754 3191 l 2753 3191 l 2752 3190 l 2752 3189 l 2752 3188 l 2752 3187 l 2752 3186 l 2752 3185 l 2752 3184 l 2752 3183 l 2752 3182 l 2752 3181 l 2752 3180 l 2752 3179 l 2752 3178 l 2752 3177 l 2752 3176 l 2752 3175 l 2752 3174 l 2753 3174 l 2753 3173 l 2754 3173 l 2755 3173 l 2755 3172 l 2755 3171 l 2755 3170 l 2755 3169 l 2755 3168 l 2755 3167 l 2755 3166 l 2755 3165 l 2755 3164 l 2755 3163 l 2755 3162 l 2755 3161 l 2755 3160 l 2755 3159 l 2754 3158 l 2753 3158 l 2752 3157 l 2752 3156 l 2752 3155 l 2752 3154 l 2752 3153 l 2752 3152 l 2752 3151 l 2752 3150 l 2752 3149 l 2752 3148 l 2752 3147 l 2752 3146 l 2752 3145 l 2752 3144 l 2752 3143 l 2752 3142 l 2752 3141 l 2752 3140 l 2752 3139 l 2752 3138 l 2752 3137 l 2752 3136 l 2752 3135 l 2752 3134 l 2752 3133 l 2752 3132 l 2752 3131 l 2752 3130 l 2752 3129 l 2752 3128 l 2753 3128 l 2754 3128 l cp gs col7 1.00 shd ef gr % Polyline n 483 3129 m 483 3130 l 484 3131 l 485 3131 l 486 3132 l 487 3132 l 488 3133 l 489 3134 l 490 3134 l 491 3134 l 492 3135 l 492 3136 l 493 3137 l 494 3137 l 495 3138 l 495 3139 l 496 3140 l 497 3140 l 498 3140 l 499 3140 l 500 3140 l 501 3141 l 502 3141 l 503 3142 l 504 3143 l 505 3143 l 506 3143 l 507 3144 l 507 3145 l 508 3146 l 509 3146 l 510 3146 l 511 3146 l 512 3146 l 513 3146 l 514 3146 l 515 3147 l 516 3147 l 516 3148 l 517 3149 l 518 3149 l 519 3149 l 520 3149 l 521 3149 l 522 3150 l 523 3150 l 524 3151 l 525 3152 l 525 3153 l 525 3154 l 526 3155 l 527 3155 l 528 3155 l 529 3155 l 530 3155 l 531 3155 l 532 3155 l 533 3155 l 534 3155 l 535 3155 l 536 3155 l 537 3155 l 538 3155 l 539 3155 l 540 3155 l 541 3155 l 542 3155 l 543 3155 l 544 3155 l 545 3155 l 546 3155 l 546 3154 l 546 3153 l 547 3153 l 547 3152 l 548 3152 l 549 3152 l 550 3152 l 551 3152 l 552 3152 l 553 3152 l 554 3152 l 555 3152 l 555 3151 l 555 3150 l 556 3150 l 556 3149 l 557 3149 l 558 3149 l 558 3148 l 558 3147 l 559 3147 l 559 3146 l 560 3146 l 561 3146 l 562 3146 l 563 3146 l 564 3146 l 565 3146 l 566 3146 l 567 3146 l 568 3145 l 569 3144 l 570 3144 l 571 3143 l 572 3143 l 573 3142 l 574 3142 l 574 3141 l 575 3140 l 577 3140 l 578 3139 l 579 3138 l 580 3137 l 581 3137 l 582 3137 l 582 3136 l 582 3135 l 583 3135 l 583 3134 l 584 3134 l 585 3134 l 586 3134 l 587 3134 l 588 3134 l 589 3134 l 590 3134 l 591 3134 l 591 3133 l 591 3132 l 592 3132 l 592 3131 l 593 3131 l 594 3131 l 595 3130 l 596 3129 l 597 3128 l 597 3127 l 597 3126 l 598 3126 l 598 3125 l 599 3125 l 600 3125 l 601 3125 l 602 3125 l 603 3125 l 604 3125 l 605 3125 l 606 3125 l 606 3124 l 606 3123 l 607 3123 l 607 3122 l 608 3122 l 609 3122 l 610 3122 l 611 3122 l 612 3122 l 613 3122 l 614 3122 l 614 3121 l 615 3121 l 615 3120 l 616 3120 l 616 3119 l 617 3119 l 618 3119 l 619 3119 l 620 3119 l 621 3119 l 622 3119 l 623 3119 l 624 3119 l 625 3119 l 626 3119 l 627 3119 l 627 3118 l 627 3117 l 628 3117 l 628 3116 l 629 3116 l 630 3116 l 631 3116 l 632 3116 l 633 3116 l 634 3116 l 634 3115 l 635 3115 l 637 3114 l 638 3114 l 639 3114 l 639 3113 l 640 3113 l 641 3113 l 642 3113 l 642 3114 l 643 3114 l 644 3115 l 645 3116 l 646 3116 l 647 3116 l 648 3116 l 649 3116 l 650 3116 l 651 3116 l 652 3116 l 653 3116 l 654 3116 l 655 3116 l 656 3116 l 658 3116 l 659 3116 l 660 3116 l 661 3116 l 662 3116 l 663 3116 l 664 3116 l 665 3116 l 666 3117 l 667 3117 l 668 3118 l 669 3119 l 670 3119 l 671 3119 l 672 3120 l 672 3121 l 673 3122 l 674 3122 l 675 3123 l 676 3123 l 677 3124 l 678 3125 l 679 3125 l 680 3125 l 681 3126 l 681 3127 l 682 3128 l 683 3128 l 684 3129 l 684 3130 l 685 3131 l 686 3131 l 687 3132 l 687 3133 l 687 3134 l 687 3135 l 687 3136 l 687 3137 l 687 3138 l 687 3139 l 687 3140 l 687 3141 l 687 3142 l 688 3143 l 689 3143 l 690 3144 l 690 3145 l 690 3146 l 690 3147 l 690 3148 l 690 3149 l 690 3150 l 690 3151 l 690 3152 l 690 3153 l 690 3154 l 690 3155 l 690 3156 l 690 3157 l 690 3158 l 690 3159 l 690 3160 l 690 3161 l 690 3162 l 690 3163 l 691 3164 l 692 3164 l 693 3165 l 693 3166 l 693 3167 l 693 3168 l 693 3169 l 693 3170 l 693 3171 l 693 3172 l 693 3173 l 693 3174 l 693 3175 l 693 3176 l 693 3177 l 693 3178 l 693 3179 l 693 3180 l 693 3181 l 693 3182 l 693 3183 l 693 3184 l 693 3185 l 693 3186 l 693 3187 l 693 3188 l 693 3189 l 693 3190 l 693 3191 l 693 3192 l 693 3193 l 693 3194 l 692 3194 l 691 3194 l 690 3194 l 689 3194 l 688 3194 l 687 3194 l 686 3194 l 685 3194 l 684 3193 l 684 3192 l 683 3191 l 682 3191 l 681 3190 l 681 3189 l 680 3188 l 679 3188 l 678 3188 l 677 3188 l 676 3188 l 675 3187 l 675 3186 l 674 3185 l 673 3185 l 672 3185 l 671 3184 l 670 3183 l 669 3183 l 668 3182 l 667 3182 l 666 3182 l 665 3182 l 664 3182 l 663 3182 l 662 3182 l 661 3182 l 660 3182 l 659 3182 l 658 3182 l 657 3182 l 656 3182 l 655 3182 l 654 3182 l 653 3182 l 652 3182 l 651 3182 l 650 3182 l 649 3182 l 648 3181 l 648 3180 l 647 3179 l 646 3179 l 645 3179 l 644 3179 l 643 3179 l 642 3179 l 641 3179 l 640 3179 l 639 3179 l 638 3179 l 637 3179 l 636 3179 l 635 3179 l 634 3179 l 633 3179 l 632 3179 l 631 3179 l 630 3179 l 629 3179 l 628 3179 l 628 3180 l 627 3180 l 627 3181 l 627 3182 l 626 3182 l 625 3182 l 624 3182 l 623 3182 l 622 3182 l 621 3182 l 620 3182 l 619 3182 l 618 3182 l 617 3182 l 616 3182 l 615 3182 l 614 3182 l 613 3182 l 613 3183 l 612 3183 l 612 3184 l 612 3185 l 611 3185 l 610 3185 l 610 3186 l 609 3186 l 609 3187 l 609 3188 l 608 3188 l 607 3188 l 606 3188 l 605 3188 l 604 3188 l 603 3188 l 602 3188 l 601 3188 l 601 3189 l 600 3189 l 600 3190 l 600 3191 l 599 3191 l 598 3191 l 598 3192 l 597 3192 l 597 3193 l 597 3194 l 596 3194 l 595 3194 l 594 3194 l 593 3194 l 592 3194 l 591 3194 l 590 3194 l 589 3194 l 588 3194 l 587 3194 l 586 3194 l 585 3194 l 584 3194 l 583 3194 l 583 3195 l 582 3195 l 582 3196 l 582 3197 l 581 3197 l 580 3197 l 579 3197 l 578 3197 l 577 3197 l 576 3198 l 575 3198 l 574 3199 l 573 3200 l 572 3200 l 571 3200 l 571 3201 l 570 3201 l 570 3202 l 570 3203 l 569 3203 l 568 3203 l 567 3203 l 566 3203 l 565 3203 l 565 3204 l 564 3204 l 564 3205 l 564 3206 l 563 3206 l 562 3206 l 561 3206 l 560 3206 l 559 3206 l 558 3207 l 557 3207 l 556 3208 l 555 3209 l 554 3209 l 553 3209 l 552 3209 l 551 3209 l 550 3209 l 549 3209 l 548 3209 l 547 3209 l 547 3210 l 546 3210 l 546 3211 l 546 3212 l 545 3212 l 544 3212 l 543 3212 l 542 3212 l 541 3212 l 540 3212 l 539 3212 l 538 3212 l 537 3212 l 536 3212 l 535 3212 l 534 3212 l 533 3212 l 532 3212 l 531 3212 l 530 3212 l 529 3212 l 528 3212 l 527 3212 l 526 3212 l 525 3212 l 524 3212 l 523 3212 l 522 3212 l 521 3212 l 520 3212 l 519 3212 l 518 3212 l 517 3212 l 516 3212 l 515 3212 l 514 3212 l 513 3211 l 513 3210 l 512 3209 l 511 3209 l 510 3208 l 510 3207 l 509 3206 l 508 3206 l 507 3205 l 507 3204 l 506 3203 l 505 3203 l 504 3202 l 504 3201 l 503 3200 l 502 3200 l 501 3199 l 501 3198 l 500 3197 l 499 3197 l 498 3197 l 497 3197 l 496 3197 l 495 3196 l 495 3195 l 494 3194 l 493 3194 l 492 3194 l 491 3194 l 490 3194 l 489 3194 l 488 3194 l 487 3194 l 486 3193 l 486 3192 l 485 3191 l 484 3191 l 483 3191 l 482 3191 l 481 3191 l 480 3190 l 480 3189 l 480 3188 l 480 3187 l 480 3186 l 480 3185 l 480 3184 l 480 3183 l 480 3182 l 480 3181 l 480 3180 l 480 3179 l 480 3178 l 480 3177 l 480 3176 l 480 3175 l 480 3174 l 481 3174 l 481 3173 l 482 3173 l 483 3173 l 483 3172 l 483 3171 l 483 3170 l 483 3169 l 483 3168 l 483 3167 l 483 3166 l 483 3165 l 483 3164 l 483 3163 l 483 3162 l 483 3161 l 483 3160 l 483 3159 l 482 3158 l 481 3158 l 480 3157 l 480 3156 l 480 3155 l 480 3154 l 480 3153 l 480 3152 l 480 3151 l 480 3150 l 480 3149 l 480 3148 l 480 3147 l 480 3146 l 480 3145 l 480 3144 l 480 3143 l 480 3142 l 480 3141 l 480 3140 l 480 3139 l 480 3138 l 480 3137 l 480 3136 l 480 3135 l 480 3134 l 480 3133 l 480 3132 l 480 3131 l 480 3130 l 480 3129 l 480 3128 l 481 3128 l 482 3128 l cp gs col7 1.00 shd ef gr % Polyline n 2755 1037 m 2755 1038 l 2756 1039 l 2757 1039 l 2758 1040 l 2759 1040 l 2760 1041 l 2761 1042 l 2762 1042 l 2763 1042 l 2764 1043 l 2764 1044 l 2765 1045 l 2766 1045 l 2767 1046 l 2767 1047 l 2768 1048 l 2769 1048 l 2770 1048 l 2771 1048 l 2772 1048 l 2773 1049 l 2774 1049 l 2775 1050 l 2776 1051 l 2777 1051 l 2778 1051 l 2779 1052 l 2779 1053 l 2780 1054 l 2781 1054 l 2782 1054 l 2783 1054 l 2784 1054 l 2785 1054 l 2786 1054 l 2787 1055 l 2788 1055 l 2788 1056 l 2789 1057 l 2790 1057 l 2791 1057 l 2792 1057 l 2793 1057 l 2794 1058 l 2795 1058 l 2796 1059 l 2797 1060 l 2797 1061 l 2797 1062 l 2798 1063 l 2799 1063 l 2800 1063 l 2801 1063 l 2802 1063 l 2803 1063 l 2804 1063 l 2805 1063 l 2806 1063 l 2807 1063 l 2808 1063 l 2809 1063 l 2810 1063 l 2811 1063 l 2812 1063 l 2813 1063 l 2814 1063 l 2815 1063 l 2816 1063 l 2817 1063 l 2818 1063 l 2818 1062 l 2818 1061 l 2819 1061 l 2819 1060 l 2820 1060 l 2821 1060 l 2822 1060 l 2823 1060 l 2824 1060 l 2825 1060 l 2826 1060 l 2827 1060 l 2827 1059 l 2827 1058 l 2828 1058 l 2828 1057 l 2829 1057 l 2830 1057 l 2830 1056 l 2830 1055 l 2831 1055 l 2831 1054 l 2832 1054 l 2833 1054 l 2834 1054 l 2835 1054 l 2836 1054 l 2837 1054 l 2838 1054 l 2839 1054 l 2840 1053 l 2841 1052 l 2842 1052 l 2843 1051 l 2844 1051 l 2845 1050 l 2846 1050 l 2846 1049 l 2847 1048 l 2849 1048 l 2850 1047 l 2851 1046 l 2852 1045 l 2853 1045 l 2854 1045 l 2854 1044 l 2854 1043 l 2855 1043 l 2855 1042 l 2856 1042 l 2857 1042 l 2858 1042 l 2859 1042 l 2860 1042 l 2861 1042 l 2862 1042 l 2863 1042 l 2863 1041 l 2863 1040 l 2864 1040 l 2864 1039 l 2865 1039 l 2866 1039 l 2867 1038 l 2868 1037 l 2869 1036 l 2869 1035 l 2869 1034 l 2870 1034 l 2870 1033 l 2871 1033 l 2872 1033 l 2873 1033 l 2874 1033 l 2875 1033 l 2876 1033 l 2877 1033 l 2878 1033 l 2878 1032 l 2878 1031 l 2879 1031 l 2879 1030 l 2880 1030 l 2881 1030 l 2882 1030 l 2883 1030 l 2884 1030 l 2885 1030 l 2886 1030 l 2886 1029 l 2887 1029 l 2887 1028 l 2888 1028 l 2888 1027 l 2889 1027 l 2890 1027 l 2891 1027 l 2892 1027 l 2893 1027 l 2894 1027 l 2895 1027 l 2896 1027 l 2897 1027 l 2898 1027 l 2899 1027 l 2899 1026 l 2899 1025 l 2900 1025 l 2900 1024 l 2901 1024 l 2902 1024 l 2903 1024 l 2904 1024 l 2905 1024 l 2906 1024 l 2906 1023 l 2907 1023 l 2909 1022 l 2910 1022 l 2911 1022 l 2911 1021 l 2912 1021 l 2913 1021 l 2914 1021 l 2914 1022 l 2915 1022 l 2916 1023 l 2917 1024 l 2918 1024 l 2919 1024 l 2920 1024 l 2921 1024 l 2922 1024 l 2923 1024 l 2924 1024 l 2925 1024 l 2926 1024 l 2927 1024 l 2928 1024 l 2930 1024 l 2931 1024 l 2932 1024 l 2933 1024 l 2934 1024 l 2935 1024 l 2936 1024 l 2937 1024 l 2938 1025 l 2939 1025 l 2940 1026 l 2941 1027 l 2942 1027 l 2943 1027 l 2944 1028 l 2944 1029 l 2945 1030 l 2946 1030 l 2947 1031 l 2948 1031 l 2949 1032 l 2950 1033 l 2951 1033 l 2952 1033 l 2953 1034 l 2953 1035 l 2954 1036 l 2955 1036 l 2956 1037 l 2956 1038 l 2957 1039 l 2958 1039 l 2959 1040 l 2959 1041 l 2959 1042 l 2959 1043 l 2959 1044 l 2959 1045 l 2959 1046 l 2959 1047 l 2959 1048 l 2959 1049 l 2959 1050 l 2960 1051 l 2961 1051 l 2962 1052 l 2962 1053 l 2962 1054 l 2962 1055 l 2962 1056 l 2962 1057 l 2962 1058 l 2962 1059 l 2962 1060 l 2962 1061 l 2962 1062 l 2962 1063 l 2962 1064 l 2962 1065 l 2962 1066 l 2962 1067 l 2962 1068 l 2962 1069 l 2962 1070 l 2962 1071 l 2963 1072 l 2964 1072 l 2965 1073 l 2965 1074 l 2965 1075 l 2965 1076 l 2965 1077 l 2965 1078 l 2965 1079 l 2965 1080 l 2965 1081 l 2965 1082 l 2965 1083 l 2965 1084 l 2965 1085 l 2965 1086 l 2965 1087 l 2965 1088 l 2965 1089 l 2965 1090 l 2965 1091 l 2965 1092 l 2965 1093 l 2965 1094 l 2965 1095 l 2965 1096 l 2965 1097 l 2965 1098 l 2965 1099 l 2965 1100 l 2965 1101 l 2965 1102 l 2964 1102 l 2963 1102 l 2962 1102 l 2961 1102 l 2960 1102 l 2959 1102 l 2958 1102 l 2957 1102 l 2956 1101 l 2956 1100 l 2955 1099 l 2954 1099 l 2953 1098 l 2953 1097 l 2952 1096 l 2951 1096 l 2950 1096 l 2949 1096 l 2948 1096 l 2947 1095 l 2947 1094 l 2946 1093 l 2945 1093 l 2944 1093 l 2943 1092 l 2942 1091 l 2941 1091 l 2940 1090 l 2939 1090 l 2938 1090 l 2937 1090 l 2936 1090 l 2935 1090 l 2934 1090 l 2933 1090 l 2932 1090 l 2931 1090 l 2930 1090 l 2929 1090 l 2928 1090 l 2927 1090 l 2926 1090 l 2925 1090 l 2924 1090 l 2923 1090 l 2922 1090 l 2921 1090 l 2920 1089 l 2920 1088 l 2919 1087 l 2918 1087 l 2917 1087 l 2916 1087 l 2915 1087 l 2914 1087 l 2913 1087 l 2912 1087 l 2911 1087 l 2910 1087 l 2909 1087 l 2908 1087 l 2907 1087 l 2906 1087 l 2905 1087 l 2904 1087 l 2903 1087 l 2902 1087 l 2901 1087 l 2900 1087 l 2900 1088 l 2899 1088 l 2899 1089 l 2899 1090 l 2898 1090 l 2897 1090 l 2896 1090 l 2895 1090 l 2894 1090 l 2893 1090 l 2892 1090 l 2891 1090 l 2890 1090 l 2889 1090 l 2888 1090 l 2887 1090 l 2886 1090 l 2885 1090 l 2885 1091 l 2884 1091 l 2884 1092 l 2884 1093 l 2883 1093 l 2882 1093 l 2882 1094 l 2881 1094 l 2881 1095 l 2881 1096 l 2880 1096 l 2879 1096 l 2878 1096 l 2877 1096 l 2876 1096 l 2875 1096 l 2874 1096 l 2873 1096 l 2873 1097 l 2872 1097 l 2872 1098 l 2872 1099 l 2871 1099 l 2870 1099 l 2870 1100 l 2869 1100 l 2869 1101 l 2869 1102 l 2868 1102 l 2867 1102 l 2866 1102 l 2865 1102 l 2864 1102 l 2863 1102 l 2862 1102 l 2861 1102 l 2860 1102 l 2859 1102 l 2858 1102 l 2857 1102 l 2856 1102 l 2855 1102 l 2855 1103 l 2854 1103 l 2854 1104 l 2854 1105 l 2853 1105 l 2852 1105 l 2851 1105 l 2850 1105 l 2849 1105 l 2848 1106 l 2847 1106 l 2846 1107 l 2845 1108 l 2844 1108 l 2843 1108 l 2843 1109 l 2842 1109 l 2842 1110 l 2842 1111 l 2841 1111 l 2840 1111 l 2839 1111 l 2838 1111 l 2837 1111 l 2837 1112 l 2836 1112 l 2836 1113 l 2836 1114 l 2835 1114 l 2834 1114 l 2833 1114 l 2832 1114 l 2831 1114 l 2830 1115 l 2829 1115 l 2828 1116 l 2827 1117 l 2826 1117 l 2825 1117 l 2824 1117 l 2823 1117 l 2822 1117 l 2821 1117 l 2820 1117 l 2819 1117 l 2819 1118 l 2818 1118 l 2818 1119 l 2818 1120 l 2817 1120 l 2816 1120 l 2815 1120 l 2814 1120 l 2813 1120 l 2812 1120 l 2811 1120 l 2810 1120 l 2809 1120 l 2808 1120 l 2807 1120 l 2806 1120 l 2805 1120 l 2804 1120 l 2803 1120 l 2802 1120 l 2801 1120 l 2800 1120 l 2799 1120 l 2798 1120 l 2797 1120 l 2796 1120 l 2795 1120 l 2794 1120 l 2793 1120 l 2792 1120 l 2791 1120 l 2790 1120 l 2789 1120 l 2788 1120 l 2787 1120 l 2786 1120 l 2785 1119 l 2785 1118 l 2784 1117 l 2783 1117 l 2782 1116 l 2782 1115 l 2781 1114 l 2780 1114 l 2779 1113 l 2779 1112 l 2778 1111 l 2777 1111 l 2776 1110 l 2776 1109 l 2775 1108 l 2774 1108 l 2773 1107 l 2773 1106 l 2772 1105 l 2771 1105 l 2770 1105 l 2769 1105 l 2768 1105 l 2767 1104 l 2767 1103 l 2766 1102 l 2765 1102 l 2764 1102 l 2763 1102 l 2762 1102 l 2761 1102 l 2760 1102 l 2759 1102 l 2758 1101 l 2758 1100 l 2757 1099 l 2756 1099 l 2755 1099 l 2754 1099 l 2753 1099 l 2752 1098 l 2752 1097 l 2752 1096 l 2752 1095 l 2752 1094 l 2752 1093 l 2752 1092 l 2752 1091 l 2752 1090 l 2752 1089 l 2752 1088 l 2752 1087 l 2752 1086 l 2752 1085 l 2752 1084 l 2752 1083 l 2752 1082 l 2753 1082 l 2753 1081 l 2754 1081 l 2755 1081 l 2755 1080 l 2755 1079 l 2755 1078 l 2755 1077 l 2755 1076 l 2755 1075 l 2755 1074 l 2755 1073 l 2755 1072 l 2755 1071 l 2755 1070 l 2755 1069 l 2755 1068 l 2755 1067 l 2754 1066 l 2753 1066 l 2752 1065 l 2752 1064 l 2752 1063 l 2752 1062 l 2752 1061 l 2752 1060 l 2752 1059 l 2752 1058 l 2752 1057 l 2752 1056 l 2752 1055 l 2752 1054 l 2752 1053 l 2752 1052 l 2752 1051 l 2752 1050 l 2752 1049 l 2752 1048 l 2752 1047 l 2752 1046 l 2752 1045 l 2752 1044 l 2752 1043 l 2752 1042 l 2752 1041 l 2752 1040 l 2752 1039 l 2752 1038 l 2752 1037 l 2752 1036 l 2753 1036 l 2754 1036 l cp gs col7 1.00 shd ef gr % Polyline 7.500 slw n 2755 1028 m 2756 1029 l 2764 1035 l 2775 1042 l 2785 1049 l 2793 1053 l 2800 1055 l 2807 1056 l 2815 1055 l 2823 1052 l 2832 1049 l 2840 1045 l 2849 1040 l 2855 1037 l 2863 1033 l 2870 1029 l 2879 1025 l 2887 1022 l 2895 1019 l 2903 1017 l 2910 1016 l 2919 1017 l 2929 1019 l 2942 1024 l 2955 1030 l 2963 1034 l 2965 1035 l gs col0 s gr % Polyline n 483 1103 m 484 1104 l 492 1110 l 503 1117 l 513 1124 l 521 1128 l 528 1130 l 535 1131 l 543 1130 l 551 1127 l 560 1124 l 568 1120 l 577 1115 l 583 1112 l 591 1108 l 598 1104 l 607 1100 l 615 1097 l 623 1094 l 631 1092 l 638 1091 l 647 1092 l 657 1094 l 670 1099 l 683 1105 l 691 1109 l 693 1110 l gs col0 s gr % Polyline n 483 1028 m 484 1029 l 492 1035 l 503 1042 l 513 1049 l 521 1053 l 528 1055 l 535 1056 l 543 1055 l 551 1052 l 560 1049 l 568 1045 l 577 1040 l 583 1037 l 591 1033 l 598 1029 l 607 1025 l 615 1022 l 623 1019 l 631 1017 l 638 1016 l 647 1017 l 657 1019 l 670 1024 l 683 1030 l 691 1034 l 693 1035 l gs col0 s gr % Polyline n 2755 1103 m 2756 1104 l 2764 1110 l 2775 1117 l 2785 1124 l 2793 1128 l 2800 1130 l 2807 1131 l 2815 1130 l 2823 1127 l 2832 1124 l 2840 1120 l 2849 1115 l 2855 1112 l 2863 1108 l 2870 1104 l 2879 1100 l 2887 1097 l 2895 1094 l 2903 1092 l 2910 1091 l 2919 1092 l 2929 1094 l 2942 1099 l 2955 1105 l 2963 1109 l 2965 1110 l gs col0 s gr % Polyline n 2755 3120 m 2756 3121 l 2764 3127 l 2775 3134 l 2785 3141 l 2793 3145 l 2800 3147 l 2807 3148 l 2815 3147 l 2823 3144 l 2832 3141 l 2840 3137 l 2849 3132 l 2855 3129 l 2863 3125 l 2870 3121 l 2879 3117 l 2887 3114 l 2895 3111 l 2903 3109 l 2910 3108 l 2919 3109 l 2929 3111 l 2942 3116 l 2955 3122 l 2963 3126 l 2965 3127 l gs col0 s gr % Polyline n 2755 3195 m 2756 3196 l 2764 3202 l 2775 3209 l 2785 3216 l 2793 3220 l 2800 3222 l 2807 3223 l 2815 3222 l 2823 3219 l 2832 3216 l 2840 3212 l 2849 3207 l 2855 3204 l 2863 3200 l 2870 3196 l 2879 3192 l 2887 3189 l 2895 3186 l 2903 3184 l 2910 3183 l 2919 3184 l 2929 3186 l 2942 3191 l 2955 3197 l 2963 3201 l 2965 3202 l gs col0 s gr % Polyline n 483 3120 m 484 3121 l 492 3127 l 503 3134 l 513 3141 l 521 3145 l 528 3147 l 535 3148 l 543 3147 l 551 3144 l 560 3141 l 568 3137 l 577 3132 l 583 3129 l 591 3125 l 598 3121 l 607 3117 l 615 3114 l 623 3111 l 631 3109 l 638 3108 l 647 3109 l 657 3111 l 670 3116 l 683 3122 l 691 3126 l 693 3127 l gs col0 s gr % Polyline n 483 3195 m 484 3196 l 492 3202 l 503 3209 l 513 3216 l 521 3220 l 528 3222 l 535 3223 l 543 3222 l 551 3219 l 560 3216 l 568 3212 l 577 3207 l 583 3204 l 591 3200 l 598 3196 l 607 3192 l 615 3189 l 623 3186 l 631 3184 l 638 3183 l 647 3184 l 657 3186 l 670 3191 l 683 3197 l 691 3201 l 693 3202 l gs col0 s gr % Polyline n 2755 1995 m 2756 1996 l 2764 2002 l 2775 2009 l 2785 2016 l 2793 2020 l 2800 2022 l 2807 2023 l 2815 2022 l 2823 2019 l 2832 2016 l 2840 2012 l 2849 2007 l 2855 2004 l 2863 2000 l 2870 1996 l 2879 1992 l 2887 1989 l 2895 1986 l 2903 1984 l 2910 1983 l 2919 1984 l 2929 1986 l 2942 1991 l 2955 1997 l 2963 2001 l 2965 2002 l gs col0 s gr % Polyline n 2755 2070 m 2756 2071 l 2764 2077 l 2775 2084 l 2785 2091 l 2793 2095 l 2800 2097 l 2807 2098 l 2815 2097 l 2823 2094 l 2832 2091 l 2840 2087 l 2849 2082 l 2855 2079 l 2863 2075 l 2870 2071 l 2879 2067 l 2887 2064 l 2895 2061 l 2903 2059 l 2910 2058 l 2919 2059 l 2929 2061 l 2942 2066 l 2955 2072 l 2963 2076 l 2965 2077 l gs col0 s gr % Polyline n 483 1995 m 484 1996 l 492 2002 l 503 2009 l 513 2016 l 521 2020 l 528 2022 l 535 2023 l 543 2022 l 551 2019 l 560 2016 l 568 2012 l 577 2007 l 583 2004 l 591 2000 l 598 1996 l 607 1992 l 615 1989 l 623 1986 l 631 1984 l 638 1983 l 647 1984 l 657 1986 l 670 1991 l 683 1997 l 691 2001 l 693 2002 l gs col0 s gr % Polyline n 483 2070 m 484 2071 l 492 2077 l 503 2084 l 513 2091 l 521 2095 l 528 2097 l 535 2098 l 543 2097 l 551 2094 l 560 2091 l 568 2087 l 577 2082 l 583 2079 l 591 2075 l 598 2071 l 607 2067 l 615 2064 l 623 2061 l 631 2059 l 638 2058 l 647 2059 l 657 2061 l 670 2066 l 683 2072 l 691 2076 l 693 2077 l gs col0 s gr $F2psEnd rs %%EndDocument @endspecial -14 2052 a Fe(Figure)23 b(1.)355 1953 y SDict begin H.S end 355 1953 a 355 1953 a SDict begin H.R end 355 1953 a 355 1953 a SDict begin [ /View [/XYZ H.V] /Dest (figure.1) cvn H.B /DEST pdfmark end 355 1953 a 99 x Fe(NetCDF)g(\002le)g(structure:)33 b(there)23 b(is)h(a)f(\002le)-14 2152 y(header)i(containing)f(metadata)h(of)g(the) g(stored)g(ar)n(\255)-14 2252 y(ra)n(ys,)f(then)d(the)i(\002x)o (ed\255siz)q(e)f(arra)n(ys)i(are)f(laid)g(out)e(in)-14 2351 y(the)e(f)n(ollo)o(wing)f(contiguous)h(\002le)h(space)g(in)g(a)g (linear)-14 2451 y(or)n(der)-5 b(,)28 b(with)f(v)n(ariab)o(le\255siz)q (e)h(arra)n(ys)g(appending)e(at)-14 2550 y(the)c(end)h(of)f(the)h (\002le)g(in)g(an)g(interlea)o(ved)g(pattern.)-114 2905 y Fg(a)h(special)f(technique)f(to)h(store)g(such)g(data.)34 b(All)24 b(record)e(v)n(ariables)-114 3004 y(share)31 b(the)h(same)g(unlimited)e(dimension)h(as)h(their)f(most)h(signif-)-114 3104 y(icant)f(dimension)g(and)f(are)i(e)o(xpected)e(to)i(gro)n(w)e (together)g(along)-114 3204 y(that)k(dimension.)64 b(The)34 b(rest,)j(less)e(signi\002cant)e(dimensions)g(all)-114 3303 y(together)g(de\002ne)i(the)f(shape)h(for)f(one)g(record)f(of)i (the)g(v)n(ariable.)-114 3403 y(F)o(or)d(\002x)o(ed-size)g(array)-5 b(,)34 b(each)e(array)f(is)j(stored)e(in)g(a)h(contiguous)-114 3503 y(\002le)23 b(space)f(starting)g(from)g(a)g(gi)n(v)o(en)f(of)n (fset.)31 b(F)o(or)22 b(v)n(ariable-size)f(ar)n(-)-114 3602 y(rays,)28 b(netCDF)f(\002rst)g(de\002nes)g(a)g Fh(r)m(ecor)m(d)f Fg(of)g(an)h(array)f(as)h(a)g(subar)n(-)-114 3702 y(ray)f(comprising)f(all)i(\002x)o(ed)f(dimensions)g(and)g(the)h (records)e(of)i(all)-114 3801 y(such)22 b(arrays)g(are)h(stored)e (interlea)n(v)o(ed)g(in)i(the)g(arrays')e(de\002ned)h(or)n(-)-114 3901 y(der)-5 b(.)53 b(Figure)309 3901 y SDict begin H.S end 309 3901 a Fg(1)351 3844 y SDict begin H.R end 351 3844 a 351 3901 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.1) cvn H.B /ANN pdfmark end 351 3901 a 30 w Fg(illustrates)30 b(the)g(storage)f(layouts)g(for)g(\002x)o(ed-size)-114 4001 y(and)20 b(v)n(ariable-size)e(arrays)i(in)g(a)h(netCDF)g(\002le.) -14 4103 y(In)45 b(order)g(to)h(achie)n(v)o(e)f(netw)o(ork)g (transparenc)o(y)e(\(machine-)-114 4202 y(independence\),)32 b(both)f(the)h(header)f(and)h(data)g(parts)g(of)f(the)i(\002le)-114 4302 y(are)c(represented)e(in)i(an)f(well-de\002ned)g(format)g(similar) h(to)f(XDR)-114 4402 y(\(eXternal)22 b(Data)h(Representation\))e(b)n (ut)i(e)o(xtended)e(to)i(support)f(ef-)-114 4501 y(\002cient)e(storage) g(of)g(arrays)f(of)h(non-byte)e(data.)-114 4630 y SDict begin H.S end -114 4630 a -114 4630 a SDict begin 12 H.A end -114 4630 a -114 4630 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.3.2) cvn H.B /DEST pdfmark end -114 4630 a 84 x Ff(3.2)90 b(Serial)23 b(NetCDF)f(API)-14 4926 y Fg(The)h(original)g(netCDF)h(API)h(w)o(as)f(designed)f(for)g(serial)i (codes)-114 5026 y(to)e(perform)d(netCDF)k(operations)d(through)f(a)j (single)g(process.)32 b(In)-114 5126 y(the)25 b(serial)g(netCDF)h (library)-5 b(,)24 b(a)i(typical)e(sequence)g(of)h(operations)-114 5225 y(to)17 b(write)g(a)g(ne)n(w)f(netCDF)h(dataset)g(is)h(to)f (create)f(the)h(dataset;)h(de\002ne)-114 5325 y(the)f(dimensions,)f(v)n (ariables)g(and)g(attrib)n(utes;)i(write)f(v)n(ariable)e(data;)2042 83 y(and)24 b(close)h(the)g(dataset.)38 b(Reading)24 b(an)g(e)o(xisting)g(netCDF)h(dataset)2042 183 y(in)m(v)n(olv)o(es)k (\002rst)h(opening)e(the)i(dataset;)35 b(inquiring)28 b(about)h(dimen-)2042 282 y(sions,)h(v)n(ariables,)g(and)d(attrib)n (utes;)32 b(reading)27 b(v)n(ariable)g(data;)32 b(and)2042 382 y(closing)20 b(the)g(dataset.)2142 482 y(These)e(netCDF)g (operations)f(can)g(be)h(di)n(vided)f(into)h(the)g(follo)n(w-)2042 582 y(ing)j(\002)n(v)o(e)h(cate)o(gories.)27 b(Refer)22 b(to)f([)3020 583 y SDict begin H.S end 3020 583 a -1 x Fg(8)3062 525 y SDict begin H.R end 3062 525 a 3062 582 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RDED97) cvn H.B /ANN pdfmark end 3062 582 a Fg(])g(for)g(details)h(of)f(each)g (function)2042 681 y(in)g(the)f(netCDF)g(library)-5 b(.)2042 750 y SDict begin H.S end 2042 750 a 2042 750 a SDict begin 12 H.A end 2042 750 a 2042 750 a SDict begin [ /View [/XYZ H.V] /Dest (Item.1) cvn H.B /DEST pdfmark end 2042 750 a 2070 865 a Fg(\(1\))40 b Fd(Dataset)28 b(Functions)p Fg(:)43 b(create/open/close)26 b(a)k(dataset,)2208 965 y(set)24 b(the)g(dataset)f(to)h(de\002ne/data)e(mode,)h(and)f(synchro-) 2208 1065 y(nize)e(dataset)h(changes)e(to)h(disk)2042 1117 y SDict begin H.S end 2042 1117 a 2042 1117 a SDict begin 12 H.A end 2042 1117 a 2042 1117 a SDict begin [ /View [/XYZ H.V] /Dest (Item.2) cvn H.B /DEST pdfmark end 2042 1117 a 2070 1232 a Fg(\(2\))40 b Fd(De\002ne)30 b(Mode)h(Functions)p Fg(:)44 b(de\002ne)29 b(dataset)h(dimen-)2208 1332 y(sions)21 b(and)e(v)n(ariables)2042 1367 y SDict begin H.S end 2042 1367 a 2042 1367 a SDict begin 12 H.A end 2042 1367 a 2042 1367 a SDict begin [ /View [/XYZ H.V] /Dest (Item.3) cvn H.B /DEST pdfmark end 2042 1367 a 2070 1499 a Fg(\(3\))40 b Fd(Attrib)n(ute)23 b(Functions)p Fg(:)31 b(manage)22 b(adding,)g(changing,)2208 1599 y(and)e(reading)f (attrib)n(utes)h(of)g(datasets)2042 1651 y SDict begin H.S end 2042 1651 a 2042 1651 a SDict begin 12 H.A end 2042 1651 a 2042 1651 a SDict begin [ /View [/XYZ H.V] /Dest (Item.4) cvn H.B /DEST pdfmark end 2042 1651 a 2070 1767 a Fg(\(4\))40 b Fd(Inquiry)52 b(Functions)p Fg(:)85 b(return)50 b(dataset)g(metadata:)2208 1866 y(dim\(id,)19 b(name,)h(len\),)f(v)n(ar\(name,)f(ndims,)i(shape,)f(id\))2042 1918 y SDict begin H.S end 2042 1918 a 2042 1918 a SDict begin 12 H.A end 2042 1918 a 2042 1918 a SDict begin [ /View [/XYZ H.V] /Dest (Item.5) cvn H.B /DEST pdfmark end 2042 1918 a 2070 2034 a Fg(\(5\))40 b Fd(Data)34 b(Access)g (Functions)p Fg(:)53 b(pro)o(vide)32 b(the)j(ability)e(to)2208 2133 y(read/write)39 b(v)n(ariable)f(data)h(in)g(one)g(of)g(the)h(\002) n(v)o(e)f(ac-)2208 2233 y(cess)17 b(methods:)22 b(single)16 b(v)n(alue,)f(whole)h(array)-5 b(,)15 b(subarray)-5 b(,)2208 2333 y(subsampled)23 b(array)h(\(strided)f(subarray\))f(and)i(mapped) 2208 2432 y(strided)c(subarray)2142 2617 y(The)c(I/O)h(implementation)e (of)h(the)h(serial)g(netCDF)g(API)g(is)g(b)n(uilt)2042 2716 y(on)33 b(the)g(nati)n(v)o(e)f(I/O)h(system)h(calls)f(and)g(has)g (its)h(o)n(wn)f(b)n(uf)n(fering)2042 2816 y(mechanism)20 b(in)h(user)f(space.)27 b(Its)22 b(design)e(and)g(optimization)f(tech-) 2042 2915 y(niques)35 b(are)g(suitable)g(for)f(serial)i(access)g(b)n (ut)f(are)g(not)g(ef)n(\002cient)2042 3015 y(or)26 b(e)n(v)o(en)g(not)g (possible)g(for)g(parallel)g(access,)i(nor)e(do)g(the)o(y)g(allo)n(w) 2042 3115 y(further)21 b(performance)e(gains)i(pro)o(vided)f(by)h (modern)g(parallel)g(I/O)2042 3214 y(techniques.)2042 3341 y SDict begin H.S end 2042 3341 a 2042 3341 a SDict begin 12 H.A end 2042 3341 a 2042 3341 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.3.3) cvn H.B /DEST pdfmark end 2042 3341 a 82 x Ff(3.3)91 b(Using)22 b(NetCDF)g(in)h(P)o(arallel)g (En)l(vir)n(onments)2142 3631 y Fg(T)-7 b(oday)34 b(most)g (scienti\002c)h(applications)e(are)i(programmed)c(to)2042 3731 y(run)25 b(in)g(parallel)g(en)m(vironments)d(due)j(to)g(the)g (increasing)f(require-)2042 3830 y(ments)30 b(on)f(data)g(amount)f(and) i(computational)d(resources.)52 b(It)30 b(is)2042 3930 y(highly)17 b(desirable)g(to)h(de)n(v)o(elop)d(a)k(set)f(of)g(parallel) f(APIs)h(for)f(access-)2042 4029 y(ing)h(netCDF)h(\002les)h(that)e (emplo)o(ys)g(appropriate)e(parallel)i(I/O)h(tech-)2042 4129 y(niques.)28 b(In)21 b(the)g(meantime,)g(programming)d(con)m(v)o (enience)g(is)k(also)2042 4229 y(important,)i(since)i(scienti\002c)f (users)g(may)f(desire)h(to)g(spend)f(mini-)2042 4328 y(mum)c(ef)n(fort)f(on)g(dealing)g(with)h(I/O)h(operations.)i(Before)d (present-)2042 4428 y(ing)f(our)f(design)g(on)h(parallel)f(netCDF)-7 b(,)20 b(we)f(w)o(ould)f(lik)o(e)h(to)g(discuss)2042 4528 y(current)26 b(approaches)f(for)h(using)h(netCDF)h(in)f(parallel)f (programs)2042 4627 y(in)21 b(a)f(message-passing)f(en)m(vironment.) 2142 4727 y(The)g(\002rst)h(and)f(most)h(straightforw)o(ard)d(approach) g(is)k(described)2042 4827 y(in)32 b(the)f(scenario)f(of)h(Figure)2930 4827 y SDict begin H.S end 2930 4827 a Fg(2)2972 4770 y SDict begin H.R end 2972 4770 a 2972 4827 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.2) cvn H.B /ANN pdfmark end 2972 4827 a Fg(\(a\))g(in)g(which)g(one)f(process)h(is)h(in)2042 4926 y(char)o(ge)18 b(of)h(collecting/distrib)n(uting)e(data)j(and)f (performing)e(I/O)i(to)2042 5026 y(a)25 b(single)f(netCDF)g(\002le)h (using)e(the)i(serial)f(netCDF)g(API.)h(The)e(I/O)2042 5126 y(requests)17 b(from)e(other)h(processes)h(are)g(carried)f(out)g (by)h(shipping)e(all)2042 5225 y(the)25 b(data)g(through)e(this)i (single)g(process.)38 b(The)25 b(dra)o(wback)d(of)j(this)2042 5325 y(approach)e(is)j(that)g(collecting)e(all)i(I/O)f(data)g(on)g(a)h (single)f(process)1928 5574 y(3)p eop %%Page: 4 4 4 3 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.4) cvn H.B /DEST pdfmark end -114 0 a -73 1021 a @beginspecial 0 @llx 0 @lly 808 @urx 204 @ury 4851 @rwi @setspecial %%BeginDocument: parallelization.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def end save -39.0 270.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 5500 m -1000 -1000 l 15112 -1000 l 15112 5500 l cp clip 0.06000 0.06000 sc % Polyline 7.500 slw n 5250 1125 m 6000 1125 l 6000 1425 l 5250 1425 l cp gs col0 s gr % Polyline n 12300 1125 m 13050 1125 l 13050 1425 l 12300 1425 l cp gs col0 s gr % Polyline n 11325 1125 m 12075 1125 l 12075 1425 l 11325 1425 l cp gs col0 s gr % Polyline n 10350 1125 m 11100 1125 l 11100 1425 l 10350 1425 l cp gs col0 s gr % Polyline n 8850 1125 m 9600 1125 l 9600 1425 l 8850 1425 l cp gs col0 s gr % Polyline n 7650 1125 m 8400 1125 l 8400 1425 l 7650 1425 l cp gs col0 s gr % Polyline n 10380 2325 m 10275 2325 10275 2670 105 arcto 4 {pop} repeat 10275 2775 13995 2775 105 arcto 4 {pop} repeat 14100 2775 14100 2430 105 arcto 4 {pop} repeat 14100 2325 10380 2325 105 arcto 4 {pop} repeat cp gs col30 1.00 shd ef gr gs col4 s gr % Polyline n 780 2325 m 675 2325 675 2670 105 arcto 4 {pop} repeat 675 2775 1620 2775 105 arcto 4 {pop} repeat 1725 2775 1725 2430 105 arcto 4 {pop} repeat 1725 2325 780 2325 105 arcto 4 {pop} repeat cp gs col2 1.00 shd ef gr gs col12 s gr % Polyline n 6450 1125 m 7200 1125 l 7200 1425 l 6450 1425 l cp gs col0 s gr % Polyline n 13275 1125 m 14025 1125 l 14025 1425 l 13275 1425 l cp gs col0 s gr % Polyline n 3750 1125 m 4500 1125 l 4500 1425 l 3750 1425 l cp gs col0 s gr % Polyline n 2775 1125 m 3525 1125 l 3525 1425 l 2775 1425 l cp gs col0 s gr % Polyline n 1800 1125 m 2550 1125 l 2550 1425 l 1800 1425 l cp gs col0 s gr % Polyline n 6405 2325 m 6300 2325 6300 2670 105 arcto 4 {pop} repeat 6300 2775 7320 2775 105 arcto 4 {pop} repeat 7425 2775 7425 2430 105 arcto 4 {pop} repeat 7425 2325 6405 2325 105 arcto 4 {pop} repeat cp gs col2 1.00 shd ef gr gs col12 s gr % Polyline n 7605 2325 m 7500 2325 7500 2670 105 arcto 4 {pop} repeat 7500 2775 8520 2775 105 arcto 4 {pop} repeat 8625 2775 8625 2430 105 arcto 4 {pop} repeat 8625 2325 7605 2325 105 arcto 4 {pop} repeat cp gs col2 1.00 shd ef gr gs col12 s gr % Polyline n 8805 2325 m 8700 2325 8700 2670 105 arcto 4 {pop} repeat 8700 2775 9720 2775 105 arcto 4 {pop} repeat 9825 2775 9825 2430 105 arcto 4 {pop} repeat 9825 2325 8805 2325 105 arcto 4 {pop} repeat cp gs col2 1.00 shd ef gr gs col12 s gr % Polyline n 5130 2325 m 5025 2325 5025 2670 105 arcto 4 {pop} repeat 5025 2775 6120 2775 105 arcto 4 {pop} repeat 6225 2775 6225 2430 105 arcto 4 {pop} repeat 6225 2325 5130 2325 105 arcto 4 {pop} repeat cp gs col2 1.00 shd ef gr gs col12 s gr % Polyline n 825 1125 m 1575 1125 l 1575 1425 l 825 1425 l cp gs col0 s gr % Polyline gs clippath 9255 2205 m 9225 2325 l 9195 2205 l 9195 2340 l 9255 2340 l cp 9195 1545 m 9225 1425 l 9255 1545 l 9255 1410 l 9195 1410 l cp clip n 9225 1425 m 9225 2325 l gs col12 s gr gr % arrowhead n 9195 1545 m 9225 1425 l 9255 1545 l 9225 1545 l 9195 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 9255 2205 m 9225 2325 l 9195 2205 l 9225 2205 l 9255 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline 45.000 slw gs clippath 1230 3387 m 1200 3507 l 1170 3387 l 1170 3660 l 1230 3660 l cp 1170 2988 m 1200 2867 l 1230 2988 l 1230 2715 l 1170 2715 l cp clip n 1200 2775 m 1200 3600 l gs col10 s gr gr % arrowhead n 1170 2988 m 1200 2867 l 1230 2988 l 1200 2988 l 1170 2988 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 1230 3387 m 1200 3507 l 1170 3387 l 1200 3387 l 1230 3387 l cp gs col10 1.00 shd ef gr col10 s % Polyline 7.500 slw gs clippath 5655 3480 m 5625 3600 l 5595 3480 l 5595 3615 l 5655 3615 l cp 5595 2895 m 5625 2775 l 5655 2895 l 5655 2760 l 5595 2760 l cp clip n 5625 2775 m 5625 3600 l gs col10 s gr gr % arrowhead n 5595 2895 m 5625 2775 l 5655 2895 l 5625 2895 l 5595 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 5655 3480 m 5625 3600 l 5595 3480 l 5625 3480 l 5655 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 6855 3480 m 6825 3600 l 6795 3480 l 6795 3615 l 6855 3615 l cp 6795 2895 m 6825 2775 l 6855 2895 l 6855 2760 l 6795 2760 l cp clip n 6825 2775 m 6825 3600 l gs col10 s gr gr % arrowhead n 6795 2895 m 6825 2775 l 6855 2895 l 6825 2895 l 6795 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 6855 3480 m 6825 3600 l 6795 3480 l 6825 3480 l 6855 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 9255 3480 m 9225 3600 l 9195 3480 l 9195 3615 l 9255 3615 l cp 9195 2895 m 9225 2775 l 9255 2895 l 9255 2760 l 9195 2760 l cp clip n 9225 2775 m 9225 3600 l gs col10 s gr gr % arrowhead n 9195 2895 m 9225 2775 l 9255 2895 l 9225 2895 l 9195 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 9255 3480 m 9225 3600 l 9195 3480 l 9225 3480 l 9255 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 10755 3480 m 10725 3600 l 10695 3480 l 10695 3615 l 10755 3615 l cp 10695 2895 m 10725 2775 l 10755 2895 l 10755 2760 l 10695 2760 l cp clip n 10725 2775 m 10725 3600 l gs col10 s gr gr % arrowhead n 10695 2895 m 10725 2775 l 10755 2895 l 10725 2895 l 10695 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 10755 3480 m 10725 3600 l 10695 3480 l 10725 3480 l 10755 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 11730 3480 m 11700 3600 l 11670 3480 l 11670 3615 l 11730 3615 l cp 11670 2895 m 11700 2775 l 11730 2895 l 11730 2760 l 11670 2760 l cp clip n 11700 2775 m 11700 3600 l gs col10 s gr gr % arrowhead n 11670 2895 m 11700 2775 l 11730 2895 l 11700 2895 l 11670 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 11730 3480 m 11700 3600 l 11670 3480 l 11700 3480 l 11730 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 12705 3480 m 12675 3600 l 12645 3480 l 12645 3615 l 12705 3615 l cp 12645 2895 m 12675 2775 l 12705 2895 l 12705 2760 l 12645 2760 l cp clip n 12675 2775 m 12675 3600 l gs col10 s gr gr % arrowhead n 12645 2895 m 12675 2775 l 12705 2895 l 12675 2895 l 12645 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 12705 3480 m 12675 3600 l 12645 3480 l 12675 3480 l 12705 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 13680 3480 m 13650 3600 l 13620 3480 l 13620 3615 l 13680 3615 l cp 13620 2895 m 13650 2775 l 13680 2895 l 13680 2760 l 13620 2760 l cp clip n 13650 2775 m 13650 3600 l gs col10 s gr gr % arrowhead n 13620 2895 m 13650 2775 l 13680 2895 l 13650 2895 l 13620 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 13680 3480 m 13650 3600 l 13620 3480 l 13650 3480 l 13680 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 8055 2205 m 8025 2325 l 7995 2205 l 7995 2340 l 8055 2340 l cp 7995 1545 m 8025 1425 l 8055 1545 l 8055 1410 l 7995 1410 l cp clip n 8025 1425 m 8025 2325 l gs col12 s gr gr % arrowhead n 7995 1545 m 8025 1425 l 8055 1545 l 8025 1545 l 7995 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 8055 2205 m 8025 2325 l 7995 2205 l 8025 2205 l 8055 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline n 10350 3600 m 14050 3600 l 14050 4000 l 10350 4000 l cp gs col0 s gr % Polyline n 5175 3600 m 9600 3600 l 9600 4000 l 5175 4000 l cp gs col0 s gr % Polyline gs clippath 8055 3480 m 8025 3600 l 7995 3480 l 7995 3615 l 8055 3615 l cp 7995 2895 m 8025 2775 l 8055 2895 l 8055 2760 l 7995 2760 l cp clip n 8025 2775 m 8025 3600 l gs col10 s gr gr % arrowhead n 7995 2895 m 8025 2775 l 8055 2895 l 8025 2895 l 7995 2895 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 8055 3480 m 8025 3600 l 7995 3480 l 8025 3480 l 8055 3480 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 6855 2205 m 6825 2325 l 6795 2205 l 6795 2340 l 6855 2340 l cp 6795 1545 m 6825 1425 l 6855 1545 l 6855 1410 l 6795 1410 l cp clip n 6825 1425 m 6825 2325 l gs col12 s gr gr % arrowhead n 6795 1545 m 6825 1425 l 6855 1545 l 6825 1545 l 6795 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 6855 2205 m 6825 2325 l 6795 2205 l 6825 2205 l 6855 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 5655 2205 m 5625 2325 l 5595 2205 l 5595 2340 l 5655 2340 l cp 5595 1545 m 5625 1425 l 5655 1545 l 5655 1410 l 5595 1410 l cp clip n 5625 1425 m 5625 2325 l gs col12 s gr gr % arrowhead n 5595 1545 m 5625 1425 l 5655 1545 l 5625 1545 l 5595 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 5655 2205 m 5625 2325 l 5595 2205 l 5625 2205 l 5655 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 4095 1545 m 4125 1425 l 4155 1545 l 4155 1410 l 4095 1410 l cp 1320 1545 m 1350 1425 l 1380 1545 l 1380 1410 l 1320 1410 l cp clip n 1350 1425 m 1350 2100 l 4125 2100 l 4125 1425 l gs col0 s gr gr % arrowhead n 1320 1545 m 1350 1425 l 1380 1545 l 1350 1545 l 1320 1545 l cp gs 0.00 setgray ef gr col0 s % arrowhead n 4095 1545 m 4125 1425 l 4155 1545 l 4125 1545 l 4095 1545 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 3120 1545 m 3150 1425 l 3180 1545 l 3180 1410 l 3120 1410 l cp 1395 1545 m 1425 1425 l 1455 1545 l 1455 1410 l 1395 1410 l cp clip n 1425 1425 m 1425 1875 l 3150 1875 l 3150 1425 l gs col0 s gr gr % arrowhead n 1395 1545 m 1425 1425 l 1455 1545 l 1425 1545 l 1395 1545 l cp gs 0.00 setgray ef gr col0 s % arrowhead n 3120 1545 m 3150 1425 l 3180 1545 l 3150 1545 l 3120 1545 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 13680 2205 m 13650 2325 l 13620 2205 l 13620 2340 l 13680 2340 l cp 13620 1545 m 13650 1425 l 13680 1545 l 13680 1410 l 13620 1410 l cp clip n 13650 1425 m 13650 2325 l gs col12 s gr gr % arrowhead n 13620 1545 m 13650 1425 l 13680 1545 l 13650 1545 l 13620 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 13680 2205 m 13650 2325 l 13620 2205 l 13650 2205 l 13680 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 12705 2205 m 12675 2325 l 12645 2205 l 12645 2340 l 12705 2340 l cp 12645 1545 m 12675 1425 l 12705 1545 l 12705 1410 l 12645 1410 l cp clip n 12675 1425 m 12675 2325 l gs col12 s gr gr % arrowhead n 12645 1545 m 12675 1425 l 12705 1545 l 12675 1545 l 12645 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 12705 2205 m 12675 2325 l 12645 2205 l 12675 2205 l 12705 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 11730 2205 m 11700 2325 l 11670 2205 l 11670 2340 l 11730 2340 l cp 11670 1545 m 11700 1425 l 11730 1545 l 11730 1410 l 11670 1410 l cp clip n 11700 1425 m 11700 2325 l gs col12 s gr gr % arrowhead n 11670 1545 m 11700 1425 l 11730 1545 l 11700 1545 l 11670 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 11730 2205 m 11700 2325 l 11670 2205 l 11700 2205 l 11730 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 10755 2205 m 10725 2325 l 10695 2205 l 10695 2340 l 10755 2340 l cp 10695 1545 m 10725 1425 l 10755 1545 l 10755 1410 l 10695 1410 l cp clip n 10725 1425 m 10725 2325 l gs col12 s gr gr % arrowhead n 10695 1545 m 10725 1425 l 10755 1545 l 10725 1545 l 10695 1545 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 10755 2205 m 10725 2325 l 10695 2205 l 10725 2205 l 10755 2205 l cp gs col12 1.00 shd ef gr col12 s % Polyline 45.000 slw gs clippath 1230 2112 m 1200 2232 l 1170 2112 l 1170 2385 l 1230 2385 l cp 1170 1638 m 1200 1517 l 1230 1638 l 1230 1365 l 1170 1365 l cp clip n 1200 1425 m 1200 2325 l gs col12 s gr gr % arrowhead n 1170 1638 m 1200 1517 l 1230 1638 l 1200 1638 l 1170 1638 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 1230 2112 m 1200 2232 l 1170 2112 l 1200 2112 l 1230 2112 l cp gs col12 1.00 shd ef gr col12 s % Polyline 7.500 slw gs clippath 2145 1545 m 2175 1425 l 2205 1545 l 2205 1410 l 2145 1410 l cp 1470 1545 m 1500 1425 l 1530 1545 l 1530 1410 l 1470 1410 l cp clip n 1500 1425 m 1500 1650 l 2175 1650 l 2175 1425 l gs col0 s gr gr % arrowhead n 1470 1545 m 1500 1425 l 1530 1545 l 1500 1545 l 1470 1545 l cp gs 0.00 setgray ef gr col0 s % arrowhead n 2145 1545 m 2175 1425 l 2205 1545 l 2175 1545 l 2145 1545 l cp gs 0.00 setgray ef gr col0 s % Polyline n 825 3600 m 4525 3600 l 4525 4000 l 825 4000 l cp gs col0 s gr /Times-Bold ff 270.00 scf sf 750 2625 m gs 1 -1 sc (netCDF) col12 sh gr /Times-Roman ff 360.00 scf sf 2325 4500 m gs 1 -1 sc (\(a\)) col0 sh gr /Times-Roman ff 360.00 scf sf 12000 4500 m gs 1 -1 sc (\(c\)) col0 sh gr /Times-Roman ff 360.00 scf sf 7200 4500 m gs 1 -1 sc (\(b\)) col0 sh gr /Times-Bold ff 285.00 scf sf 6375 2625 m gs 1 -1 sc (netCDF) col12 sh gr /Times-Bold ff 285.00 scf sf 7575 2625 m gs 1 -1 sc (netCDF) col12 sh gr /Times-Bold ff 285.00 scf sf 8775 2625 m gs 1 -1 sc (netCDF) col12 sh gr /Times-Bold ff 285.00 scf sf 5175 2625 m gs 1 -1 sc (netCDF) col12 sh gr /Times-Roman ff 300.00 scf sf 1425 3900 m gs 1 -1 sc (Parallel File System) col10 sh gr /Times-Roman ff 240.00 scf sf 3050 1350 m gs 1 -1 sc (P2) col0 sh gr /Times-Roman ff 240.00 scf sf 3975 1350 m gs 1 -1 sc (P3) col0 sh gr /Times-Roman ff 240.00 scf sf 6700 1350 m gs 1 -1 sc (P1) col0 sh gr /Times-Roman ff 240.00 scf sf 5500 1350 m gs 1 -1 sc (P0) col0 sh gr /Times-Roman ff 240.00 scf sf 7925 1350 m gs 1 -1 sc (P2) col0 sh gr /Times-Roman ff 240.00 scf sf 10625 1350 m gs 1 -1 sc (P0) col0 sh gr /Times-Roman ff 240.00 scf sf 11575 1350 m gs 1 -1 sc (P1) col0 sh gr /Times-Roman ff 240.00 scf sf 12575 1350 m gs 1 -1 sc (P2) col0 sh gr /Times-Roman ff 240.00 scf sf 13550 1350 m gs 1 -1 sc (P3) col0 sh gr /Times-Roman ff 300.00 scf sf 6225 3900 m gs 1 -1 sc (Parallel File System) col10 sh gr /Times-Roman ff 300.00 scf sf 11100 3900 m gs 1 -1 sc (Parallel File System) col10 sh gr /Times-Bold ff 270.00 scf sf 11325 2625 m gs 1 -1 sc (Parallel netCDF) col4 sh gr /Times-Roman ff 240.00 scf sf 9125 1350 m gs 1 -1 sc (P3) col0 sh gr /Times-Roman ff 240.00 scf sf 1100 1350 m gs 1 -1 sc (P0) col0 sh gr /Times-Roman ff 240.00 scf sf 2050 1350 m gs 1 -1 sc (P1) col0 sh gr $F2psEnd rs %%EndDocument @endspecial -14 1226 a Fe(Figure)20 b(2.)349 1127 y SDict begin H.S end 349 1127 a 349 1127 a SDict begin H.R end 349 1127 a 349 1127 a SDict begin [ /View [/XYZ H.V] /Dest (figure.2) cvn H.B /DEST pdfmark end 349 1127 a 99 x Fe(Using)g(netCDF)g(in)g(parallel)h(pr)n(ograms:)30 b(\(a\))20 b(use)h(serial)g(netCDF)e(API)i(to)f(access)h(single)f (\002les)g(thr)n(ough)-14 1326 y(a)j(single)f(pr)n(ocess;)i(\(b\))d (use)i(serial)h(netCDF)e(API)h(to)f(access)i(m)n(ultiple)e(\002les)h (concurrentl)o(y)g(and)f(independentl)o(y;)-14 1426 y(\(c\))g(use)h(ne) o(w)g(parallel)g(netCDF)g(API)g(to)g(access)h(single)f(\002les)g (cooperativel)o(y)g(or)g(collectivel)o(y)-7 b(.)-114 1776 y Fg(can)32 b(easily)f(cause)h(an)g(I/O)g(performance)c (bottleneck)j(and)g(may)-114 1875 y(o)o(v)o(erwhelm)18 b(its)j(memory)d(capacity)-5 b(.)-14 2010 y(T)e(o)30 b(a)n(v)n(oid)h(unnecessary)d(data)j(shipping,)g(an)g(alternati)n(v)o (e)e(ap-)-114 2110 y(proach)c(is)j(to)f(ha)n(v)o(e)g(all)g(processes)g (perform)d(their)j(I/O)g(indepen-)-114 2209 y(dently)14 b(using)h(the)g(serial)h(netCDF)g(API,)f(as)h(sho)n(wn)f(in)g(Figure) 1696 2209 y SDict begin H.S end 1696 2209 a Fg(2)1737 2153 y SDict begin H.R end 1737 2153 a 1737 2209 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.2) cvn H.B /ANN pdfmark end 1737 2209 a Fg(\(b\).)-114 2309 y(In)35 b(this)i(case,)j (all)c(netCDF)g(operations)e(can)i(proceed)e(concur)n(-)-114 2409 y(rently)-5 b(,)25 b(b)n(ut)g(o)o(v)o(er)f(multiple)g(\002les,)k (one)c(for)h(each)f(process.)40 b(Ho)n(w-)-114 2508 y(e)n(v)o(er)m(,)29 b(using)f(multiple)f(\002les)j(to)e(store)g(a)h(single)f(netCDF)h (dataset)-114 2608 y(results)g(the)f(comple)o(xity)f(and)h(dif)n (\002culty)f(of)h(data)h(management.)-114 2708 y(This)21 b(approach)f(also)h(destructs)g(the)g(purpose)f(of)h(netCDF)h(design) -114 2807 y(on)e(easy)g(data)g(inte)o(gration)e(and)i(management.)-14 2942 y(A)i(third)f(approach)e(introduces)h(a)i(ne)n(w)g(set)g(of)g (APIs)g(with)g(par)n(-)-114 3042 y(allel)16 b(access)h(semantics)f(and) f(optimized)g(parallel)g(I/O)h(implemen-)-114 3141 y(tation)23 b(such)f(that)i(all)f(processes)g(perform)e(I/O)i(operations)f(coop-) -114 3241 y(erati)n(v)o(ely)c(or)g(collecti)n(v)o(ely)f(through)g(the)i (parallel)f(netCDF)h(library)-114 3341 y(to)k(access)h(a)f(single)g (netCDF)g(\002le.)34 b(This)23 b(approach,)e(as)j(sho)n(wn)e(in)-114 3440 y(Figure)120 3440 y SDict begin H.S end 120 3440 a Fg(2)162 3384 y SDict begin H.R end 162 3384 a 162 3440 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.2) cvn H.B /ANN pdfmark end 162 3440 a Fg(\(c\),)17 b(both)f(frees)h(the)g(users) h(from)e(dealing)g(with)h(details)h(of)-114 3540 y(parallel)i(I/O)g (and)g(pro)o(vides)e(more)h(opportunities)f(for)h(emplo)o(ying)-114 3639 y(v)n(arious)j(parallel)h(I/O)g(optimizations)f(in)i(order)e(to)h (obtain)g(higher)-114 3739 y(performance.)f(W)-7 b(e)19 b(discuss)h(the)f(details)g(of)f(this)h(parallel)g(netCDF)-114 3839 y(design)h(and)f(implementation)f(in)i(the)h(ne)o(xt)e(section.) -114 4008 y SDict begin H.S end -114 4008 a -114 4008 a SDict begin 12 H.A end -114 4008 a -114 4008 a SDict begin [ /View [/XYZ H.V] /Dest (section.4) cvn H.B /DEST pdfmark end -114 4008 a 134 x Fi(4)99 b(P)o(arallel)24 b(NetCDF)-14 4428 y Fg(T)-7 b(o)33 b(f)o(acilitate)h(con)m(v)o(enient)d(and)i (high-performance)c(parallel)-114 4528 y(access)g(to)f(netCDF)g (\002les,)j(we)d(de\002ne)g(a)g(ne)n(w)g(parallel)f(interf)o(ace)-114 4628 y(and)15 b(pro)o(vide)f(a)i(prototype)e(implementation.)21 b(Since)16 b(a)g(lar)o(ge)e(num-)-114 4727 y(ber)37 b(of)g(e)o(xisting) g(users)g(are)g(running)e(their)j(applications)e(o)o(v)o(er)-114 4827 y(netCDF)-7 b(,)39 b(our)e(parallel)h(netCDF)h(design)e(retains)h (the)g(original)-114 4926 y(netCDF)e(\002le)f(format)f(\(v)o(ersion)g (3\))h(and)f(introduces)g(minimum)-114 5026 y(changes)18 b(from)g(the)h(original)f(interf)o(ace.)24 b(W)-7 b(e)20 b(distinguish)f(the)g(par)n(-)-114 5126 y(allel)32 b(API)f(from)f(the)i (original)e(serial)h(API)h(by)f(pre\002xing)e(the)i(C)-114 5225 y(function)21 b(calls)j(with)f(\223ncmpi)p 784 5225 25 4 v 28 w(\224)h(and)e(the)h(F)o(ortran)e(function)h(calls)-114 5325 y(with)e(\223nfmpi)p 296 5325 V 28 w(\224.)2042 1693 y SDict begin H.S end 2042 1693 a 2042 1693 a SDict begin 12 H.A end 2042 1693 a 2042 1693 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.4.1) cvn H.B /DEST pdfmark end 2042 1693 a 83 x Ff(4.1)91 b(Interface)21 b(Design)2142 2001 y Fg(Our)g(parallel)f(netCDF)i(API)g(is)g(b)n(uilt)f(on)g(top)g (of)g(MPI-IO.)f(The)2042 2101 y(parallel)27 b(netCDF)i(b)n(uilt)e(on)h (MPI-IO)f(can)g(bene\002t)h(from)e(se)n(v)o(eral)2042 2200 y(well-kno)n(wn)21 b(optimizations)f(already)h(used)h(in)h(e)o (xisting)e(MPI-IO)2042 2300 y(implementations,)32 b(such)f(as)h(data)g (sie)n(ving)e(and)h(tw)o(o-phase)f(I/O)2042 2400 y(strate)o(gies)h([) 2414 2400 y SDict begin H.S end 2414 2400 a Fg(11)2496 2343 y SDict begin H.R end 2496 2343 a 2496 2400 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.RoBC93) cvn H.B /ANN pdfmark end 2496 2400 a Fg(,)2548 2400 y SDict begin H.S end 2548 2400 a Fg(12)2631 2343 y SDict begin H.R end 2631 2343 a 2631 2400 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.TBCP94) cvn H.B /ANN pdfmark end 2631 2400 a Fg(,)2683 2401 y SDict begin H.S end 2683 2401 a -1 x Fg(13)2766 2343 y SDict begin H.R end 2766 2343 a 2766 2400 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThCh96) cvn H.B /ANN pdfmark end 2766 2400 a Fg(,)2818 2401 y SDict begin H.S end 2818 2401 a -1 x Fg(15)2901 2343 y SDict begin H.R end 2901 2343 a 2901 2400 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThGL99) cvn H.B /ANN pdfmark end 2901 2400 a Fg(])g(in)g(R)m(OMIO.)f(Figure)3629 2401 y SDict begin H.S end 3629 2401 a -1 x Fg(3)3671 2343 y SDict begin H.R end 3671 2343 a 3671 2400 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.3) cvn H.B /ANN pdfmark end 3671 2400 a 31 w Fg(describes)2042 2499 y(the)20 b(o)o(v)o(erall)f(architecture)g(for)g(our)h(design.)2142 2608 y(In)f(parallel)f(netCDF)-7 b(,)20 b(a)f(\002le)h(is)g(opened,)d (operated,)g(and)i(closed)2042 2708 y(by)d(the)h(participating)e (processes)h(in)h(a)g(communication)d(group.)22 b(In)2042 2807 y(order)g(for)g(these)h(processes)f(to)h(operate)f(on)g(the)h (same)f(\002le)i(space,)2042 2907 y(especially)33 b(the)g(structural)f (information)e(contained)i(in)h(the)g(\002le)2042 3006 y(header)m(,)28 b(a)g(number)e(of)i(changes)f(ha)n(v)o(e)g(been)g(made) g(to)h(the)g(orig-)2042 3106 y(inal)21 b(serial)f(netCDF)h(API.)2142 3215 y(F)o(or)h(the)h(function)f(calls)h(that)g(create/open)f(a)h (netCDF)h(\002le,)g(an)2042 3314 y(MPI)j(communicator)c(is)k(added)e (in)h(the)h(ar)o(gument)c(list)28 b(to)e(de\002ne)2042 3414 y(the)33 b(participating)d(I/O)i(processes)g(within)g(the)h (\002le')-5 b(s)33 b(open)e(and)2042 3514 y(close)22 b(scope.)31 b(MPI)p 2630 3514 25 4 v 29 w(Info)21 b(object)h(is)h(also) f(added)f(to)h(pass)h(user)f(ac-)2042 3613 y(cess)k(hints)f(to)f(the)h (MPI-IO)f(for)g(further)g(optimizations.)37 b(By)25 b(de-)2042 3713 y(scribing)j(the)g(collection)g(of)g(processes)g(with)h(a)g (communicator)m(,)2042 3813 y(we)16 b(pro)o(vide)e(the)i(underlying)d (implementation)g(with)j(information)2042 3912 y(that)27 b(can)g(be)g(used)f(to)h(ensure)f(\002le)i(consistenc)o(y)-5 b(.)43 b(The)26 b(MPI)p 3848 3912 V 30 w(Info)2042 4012 y(hint)g(pro)o(vides)f(users)i(the)f(ability)h(to)f(deli)n(v)o(er)f (the)i(high)f(le)n(v)o(el)g(ac-)2042 4111 y(cess)h(information)d(to)i (netCDF)g(and)g(MPI-IO)f(libraries,)i(such)f(as)2042 4211 y(\002le)g(access)h(patterns)d(and)h(\002le)i(system)e (speci\002cs)h(to)g(direct)f(opti-)2042 4311 y(mization.)2142 4419 y(W)-7 b(e)25 b(k)o(eep)f(the)g(same)g(syntax)f(and)h(semantics)g (for)f(the)h(parallel)2042 4519 y(netCDF)g(de\002ne)g(mode)e (functions,)h(attrib)n(ute)h(functions,)f(and)g(in-)2042 4619 y(quiry)30 b(functions)f(as)i(the)f(original)g(ones.)55 b(These)31 b(functions)e(are)2042 4718 y(also)38 b(made)e(collecti)n(v) o(e)g(to)h(guarantee)e(consistenc)o(y)h(of)g(dataset)2042 4818 y(structure)15 b(among)f(the)h(participating)f(processes)h(in)h (the)g(same)f(MPI)2042 4917 y(communication)k(group.)29 b(F)o(or)22 b(instance,)f(the)h(de\002ne)g(mode)f(func-)2042 5017 y(tions)e(is)h(required)c(to)j(be)g(called)f(by)h(the)f(processes) h(with)g(the)f(same)2042 5117 y(v)n(alues.)2142 5225 y(The)33 b(major)f(ef)n(fort)g(of)h(this)h(w)o(ork)f(is)h(the)f (parallelization)f(of)2042 5325 y(the)25 b(data)g(access)g(functions.) 38 b(W)-7 b(e)26 b(pro)o(vide)d(tw)o(o)i(sets)h(of)f(data)g(ac-)1928 5574 y(4)p eop %%Page: 5 5 5 4 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.5) cvn H.B /DEST pdfmark end -114 0 a -58 1407 a @beginspecial 0 @llx 0 @lly 376 @urx 285 @ury 2227 @rwi @setspecial %%BeginDocument: hierarchy.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def end save -3.0 302.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /reencdict 12 dict def /ReEncode { reencdict begin /newcodesandnames exch def /newfontname exch def /basefontname exch def /basefontdict basefontname findfont def /newfont basefontdict maxlength dict def basefontdict { exch dup /FID ne { dup /Encoding eq { exch dup length array copy newfont 3 1 roll put } { exch newfont 3 1 roll put } ifelse } { pop pop } ifelse } forall newfont /FontName newfontname put newcodesandnames aload pop 128 1 255 { newfont /Encoding get exch /.notdef put } for newcodesandnames length 2 idiv { newfont /Encoding get 3 1 roll put } repeat newfontname newfont definefont pop end } def /isovec [ 8#200 /grave 8#201 /acute 8#202 /circumflex 8#203 /tilde 8#204 /macron 8#205 /breve 8#206 /dotaccent 8#207 /dieresis 8#210 /ring 8#211 /cedilla 8#212 /hungarumlaut 8#213 /ogonek 8#214 /caron 8#220 /dotlessi 8#230 /oe 8#231 /OE 8#240 /space 8#241 /exclamdown 8#242 /cent 8#243 /sterling 8#244 /currency 8#245 /yen 8#246 /brokenbar 8#247 /section 8#250 /dieresis 8#251 /copyright 8#252 /ordfeminine 8#253 /guillemotleft 8#254 /logicalnot 8#255 /endash 8#256 /registered 8#257 /macron 8#260 /degree 8#261 /plusminus 8#262 /twosuperior 8#263 /threesuperior 8#264 /acute 8#265 /mu 8#266 /paragraph 8#267 /periodcentered 8#270 /cedilla 8#271 /onesuperior 8#272 /ordmasculine 8#273 /guillemotright 8#274 /onequarter 8#275 /onehalf 8#276 /threequarters 8#277 /questiondown 8#300 /Agrave 8#301 /Aacute 8#302 /Acircumflex 8#303 /Atilde 8#304 /Adieresis 8#305 /Aring 8#306 /AE 8#307 /Ccedilla 8#310 /Egrave 8#311 /Eacute 8#312 /Ecircumflex 8#313 /Edieresis 8#314 /Igrave 8#315 /Iacute 8#316 /Icircumflex 8#317 /Idieresis 8#320 /Eth 8#321 /Ntilde 8#322 /Ograve 8#323 /Oacute 8#324 /Ocircumflex 8#325 /Otilde 8#326 /Odieresis 8#327 /multiply 8#330 /Oslash 8#331 /Ugrave 8#332 /Uacute 8#333 /Ucircumflex 8#334 /Udieresis 8#335 /Yacute 8#336 /Thorn 8#337 /germandbls 8#340 /agrave 8#341 /aacute 8#342 /acircumflex 8#343 /atilde 8#344 /adieresis 8#345 /aring 8#346 /ae 8#347 /ccedilla 8#350 /egrave 8#351 /eacute 8#352 /ecircumflex 8#353 /edieresis 8#354 /igrave 8#355 /iacute 8#356 /icircumflex 8#357 /idieresis 8#360 /eth 8#361 /ntilde 8#362 /ograve 8#363 /oacute 8#364 /ocircumflex 8#365 /otilde 8#366 /odieresis 8#367 /divide 8#370 /oslash 8#371 /ugrave 8#372 /uacute 8#373 /ucircumflex 8#374 /udieresis 8#375 /yacute 8#376 /thorn 8#377 /ydieresis] def /Times-Roman /Times-Roman-iso isovec ReEncode /Times-Bold /Times-Bold-iso isovec ReEncode /DrawEllipse { /endangle exch def /startangle exch def /yrad exch def /xrad exch def /y exch def /x exch def /savematrix mtrx currentmatrix def x y tr xrad yrad sc 0 0 1 startangle endangle arc closepath savematrix setmatrix } def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 6033 m -1000 -1000 l 7312 -1000 l 7312 6033 l cp clip 0.06000 0.06000 sc % Polyline 7.500 slw n 1455 1275 m 1350 1275 1350 2145 105 arcto 4 {pop} repeat 1350 2250 5745 2250 105 arcto 4 {pop} repeat 5850 2250 5850 1380 105 arcto 4 {pop} repeat 5850 1275 1455 1275 105 arcto 4 {pop} repeat cp gs col30 1.00 shd ef gr gs col4 s gr % Polyline n 2280 1725 m 2175 1725 2175 2070 105 arcto 4 {pop} repeat 2175 2175 5670 2175 105 arcto 4 {pop} repeat 5775 2175 5775 1830 105 arcto 4 {pop} repeat 5775 1725 2280 1725 105 arcto 4 {pop} repeat cp gs col11 1.00 shd ef gr gs col10 s gr % Ellipse n 1800 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr % Polyline gs clippath 4230 1155 m 4200 1275 l 4170 1155 l 4170 1290 l 4230 1290 l cp 4170 945 m 4200 825 l 4230 945 l 4230 810 l 4170 810 l cp clip n 4200 825 m 4200 1275 l gs col12 s gr gr % arrowhead n 4170 945 m 4200 825 l 4230 945 l 4200 945 l 4170 945 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 4230 1155 m 4200 1275 l 4170 1155 l 4200 1155 l 4230 1155 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 2655 2527 m 2625 2647 l 2595 2527 l 2595 2662 l 2655 2662 l cp 2595 2370 m 2625 2250 l 2655 2370 l 2655 2235 l 2595 2235 l cp clip n 2625 2250 m 2625 2647 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 2595 2370 m 2625 2250 l 2655 2370 l 2625 2370 l 2595 2370 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 2655 2527 m 2625 2647 l 2595 2527 l 2625 2527 l 2655 2527 l cp gs col10 1.00 shd ef gr col10 s % Ellipse n 2325 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr % Polyline [60 30 15 30] 0 sd n 75 1950 m 2175 1950 l gs col0 s gr [] 0 sd % Polyline n 1350 3600 m 2250 3600 l 2250 4125 l 1350 4125 l cp gs col0 s gr /Times-Bold-iso ff 210.00 scf sf 2325 1575 m gs 1 -1 sc (Parallel netCDF) dup sw pop 2 div neg 0 rm col4 sh gr /Times-Roman-iso ff 210.00 scf sf 150 1875 m gs 1 -1 sc (User Space) col0 sh gr % Ellipse n 1275 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 375 2400 m gs 1 -1 sc (Space) col0 sh gr /Times-Roman-iso ff 210.00 scf sf 3600 3000 m gs 1 -1 sc (Communication Network) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman-iso ff 210.00 scf sf 150 2175 m gs 1 -1 sc (File System) col0 sh gr /Times-Roman-iso ff 210.00 scf sf 5400 4050 m gs 1 -1 sc (Server) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 3075 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 5400 3825 m gs 1 -1 sc (I/O) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 5925 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 3600 4050 m gs 1 -1 sc (Server) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Roman-iso ff 210.00 scf sf 3600 3825 m gs 1 -1 sc (I/O) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 3600 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 1800 4050 m gs 1 -1 sc (Server) dup sw pop 2 div neg 0 rm col0 sh gr % Polyline gs clippath 1770 4245 m 1800 4125 l 1830 4245 l 1830 4110 l 1770 4110 l cp 1830 4455 m 1800 4575 l 1770 4455 l 1770 4590 l 1830 4590 l cp clip n 1800 4575 m 1800 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 1830 4455 m 1800 4575 l 1770 4455 l 1800 4455 l 1830 4455 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 1770 4245 m 1800 4125 l 1830 4245 l 1800 4245 l 1770 4245 l cp gs col10 1.00 shd ef gr col10 s /Times-Roman-iso ff 210.00 scf sf 1800 3825 m gs 1 -1 sc (I/O) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Roman-iso ff 210.00 scf sf 5400 750 m gs 1 -1 sc (Node) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 4125 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 5400 525 m gs 1 -1 sc (Compute) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 3600 2925 2025 300 0 360 DrawEllipse gs col0 s gr /Times-Roman-iso ff 210.00 scf sf 4200 750 m gs 1 -1 sc (Node) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Roman-iso ff 210.00 scf sf 4200 525 m gs 1 -1 sc (Compute) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 4875 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 3000 750 m gs 1 -1 sc (Node) dup sw pop 2 div neg 0 rm col0 sh gr % Polyline gs clippath 3283 4208 m 3375 4125 l 3333 4241 l 3408 4129 l 3358 4096 l cp 3167 4492 m 3075 4575 l 3117 4459 l 3042 4571 l 3092 4604 l cp clip n 3075 4575 m 3375 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 3167 4492 m 3075 4575 l 3117 4459 l 3142 4475 l 3167 4492 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3283 4208 m 3375 4125 l 3333 4241 l 3308 4225 l 3283 4208 l cp gs col10 1.00 shd ef gr col10 s /Times-Roman-iso ff 210.00 scf sf 3000 525 m gs 1 -1 sc (Compute) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Roman-iso ff 210.00 scf sf 1800 750 m gs 1 -1 sc (Node) dup sw pop 2 div neg 0 rm col0 sh gr % Ellipse n 5400 4950 225 75 0 360 DrawEllipse gs col7 1.00 shd ef gr gs col-1 s gr /Times-Roman-iso ff 210.00 scf sf 1800 525 m gs 1 -1 sc (Compute) dup sw pop 2 div neg 0 rm col0 sh gr % Polyline gs clippath 3570 4245 m 3600 4125 l 3630 4245 l 3630 4110 l 3570 4110 l cp 3630 4455 m 3600 4575 l 3570 4455 l 3570 4590 l 3630 4590 l cp clip n 3600 4575 m 3600 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 3630 4455 m 3600 4575 l 3570 4455 l 3600 4455 l 3630 4455 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3570 4245 m 3600 4125 l 3630 4245 l 3600 4245 l 3570 4245 l cp gs col10 1.00 shd ef gr col10 s % Polyline n 3150 3600 m 4050 3600 l 4050 4125 l 3150 4125 l cp gs col0 s gr % Polyline [60 30 15 30] 0 sd n 5775 1950 m 6300 1950 l gs col0 s gr [] 0 sd % Polyline n 4950 3600 m 5850 3600 l 5850 4125 l 4950 4125 l cp gs col0 s gr % Polyline gs clippath 1483 4208 m 1575 4125 l 1533 4241 l 1608 4129 l 1558 4096 l cp 1367 4492 m 1275 4575 l 1317 4459 l 1242 4571 l 1292 4604 l cp clip n 1275 4575 m 1575 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 1367 4492 m 1275 4575 l 1317 4459 l 1342 4475 l 1367 4492 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 1483 4208 m 1575 4125 l 1533 4241 l 1508 4225 l 1483 4208 l cp gs col10 1.00 shd ef gr col10 s % Polyline n 1350 300 m 2250 300 l 2250 825 l 1350 825 l cp gs col0 s gr % Polyline gs clippath 5370 4245 m 5400 4125 l 5430 4245 l 5430 4110 l 5370 4110 l cp 5430 4455 m 5400 4575 l 5370 4455 l 5370 4590 l 5430 4590 l cp clip n 5400 4575 m 5400 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 5430 4455 m 5400 4575 l 5370 4455 l 5400 4455 l 5430 4455 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 5370 4245 m 5400 4125 l 5430 4245 l 5400 4245 l 5370 4245 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 2127 4246 m 2100 4125 l 2180 4219 l 2120 4098 l 2066 4125 l cp 2298 4454 m 2325 4575 l 2245 4481 l 2305 4602 l 2359 4575 l cp clip n 2325 4575 m 2100 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 2298 4454 m 2325 4575 l 2245 4481 l 2271 4468 l 2298 4454 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 2127 4246 m 2100 4125 l 2180 4219 l 2154 4232 l 2127 4246 l cp gs col10 1.00 shd ef gr col10 s % Polyline n 2550 300 m 3450 300 l 3450 825 l 2550 825 l cp gs col0 s gr % Polyline gs clippath 1830 1155 m 1800 1275 l 1770 1155 l 1770 1290 l 1830 1290 l cp 1770 945 m 1800 825 l 1830 945 l 1830 810 l 1770 810 l cp clip n 1800 825 m 1800 1275 l gs col12 s gr gr % arrowhead n 1770 945 m 1800 825 l 1830 945 l 1800 945 l 1770 945 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 1830 1155 m 1800 1275 l 1770 1155 l 1800 1155 l 1830 1155 l cp gs col12 1.00 shd ef gr col12 s % Polyline gs clippath 3030 1155 m 3000 1275 l 2970 1155 l 2970 1290 l 3030 1290 l cp 2970 945 m 3000 825 l 3030 945 l 3030 810 l 2970 810 l cp clip n 3000 825 m 3000 1275 l gs col12 s gr gr % arrowhead n 2970 945 m 3000 825 l 3030 945 l 3000 945 l 2970 945 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 3030 1155 m 3000 1275 l 2970 1155 l 3000 1155 l 3030 1155 l cp gs col12 1.00 shd ef gr col12 s % Polyline n 3750 300 m 4650 300 l 4650 825 l 3750 825 l cp gs col0 s gr % Polyline gs clippath 5430 1155 m 5400 1275 l 5370 1155 l 5370 1290 l 5430 1290 l cp 5370 945 m 5400 825 l 5430 945 l 5430 810 l 5370 810 l cp clip n 5400 825 m 5400 1275 l gs col12 s gr gr % arrowhead n 5370 945 m 5400 825 l 5430 945 l 5400 945 l 5370 945 l cp gs col12 1.00 shd ef gr col12 s % arrowhead n 5430 1155 m 5400 1275 l 5370 1155 l 5400 1155 l 5430 1155 l cp gs col12 1.00 shd ef gr col12 s % Polyline n 4950 300 m 5850 300 l 5850 825 l 4950 825 l cp gs col0 s gr % Polyline gs clippath 3927 4246 m 3900 4125 l 3980 4219 l 3920 4098 l 3866 4125 l cp 4098 4454 m 4125 4575 l 4045 4481 l 4105 4602 l 4159 4575 l cp clip n 4125 4575 m 3900 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 4098 4454 m 4125 4575 l 4045 4481 l 4071 4468 l 4098 4454 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3927 4246 m 3900 4125 l 3980 4219 l 3954 4232 l 3927 4246 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 5727 4246 m 5700 4125 l 5780 4219 l 5720 4098 l 5666 4125 l cp 5898 4454 m 5925 4575 l 5845 4481 l 5905 4602 l 5959 4575 l cp clip n 5925 4575 m 5700 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 5898 4454 m 5925 4575 l 5845 4481 l 5871 4468 l 5898 4454 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 5727 4246 m 5700 4125 l 5780 4219 l 5754 4232 l 5727 4246 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 4878 3246 m 4800 3150 l 4914 3198 l 4806 3117 l 4770 3165 l cp 5322 3504 m 5400 3600 l 5286 3552 l 5394 3633 l 5430 3585 l cp clip n 5400 3600 m 4800 3150 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 5322 3504 m 5400 3600 l 5286 3552 l 5304 3528 l 5322 3504 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 4878 3246 m 4800 3150 l 4914 3198 l 4896 3222 l 4878 3246 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 5083 4208 m 5175 4125 l 5133 4241 l 5208 4129 l 5158 4096 l cp 4967 4492 m 4875 4575 l 4917 4459 l 4842 4571 l 4892 4604 l cp clip n 4875 4575 m 5175 4125 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 4967 4492 m 4875 4575 l 4917 4459 l 4942 4475 l 4967 4492 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 5083 4208 m 5175 4125 l 5133 4241 l 5108 4225 l 5083 4208 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 3330 2505 m 3300 2625 l 3270 2505 l 3270 2640 l 3330 2640 l cp 3270 2370 m 3300 2250 l 3330 2370 l 3330 2235 l 3270 2235 l cp clip n 3300 2250 m 3300 2625 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 3270 2370 m 3300 2250 l 3330 2370 l 3300 2370 l 3270 2370 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3330 2505 m 3300 2625 l 3270 2505 l 3300 2505 l 3330 2505 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 3930 2505 m 3900 2625 l 3870 2505 l 3870 2640 l 3930 2640 l cp 3870 2370 m 3900 2250 l 3930 2370 l 3930 2235 l 3870 2235 l cp clip n 3900 2250 m 3900 2625 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 3870 2370 m 3900 2250 l 3930 2370 l 3900 2370 l 3870 2370 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3930 2505 m 3900 2625 l 3870 2505 l 3900 2505 l 3930 2505 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 4605 2535 m 4575 2655 l 4545 2535 l 4545 2670 l 4605 2670 l cp 4545 2370 m 4575 2250 l 4605 2370 l 4605 2235 l 4545 2235 l cp clip n 4575 2250 m 4575 2655 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 4545 2370 m 4575 2250 l 4605 2370 l 4575 2370 l 4545 2370 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 4605 2535 m 4575 2655 l 4545 2535 l 4575 2535 l 4605 2535 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 2286 3198 m 2400 3150 l 2322 3246 l 2430 3165 l 2394 3117 l cp 1914 3552 m 1800 3600 l 1878 3504 l 1770 3585 l 1806 3633 l cp clip n 1800 3600 m 2400 3150 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 1914 3552 m 1800 3600 l 1878 3504 l 1896 3528 l 1914 3552 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 2286 3198 m 2400 3150 l 2322 3246 l 2304 3222 l 2286 3198 l cp gs col10 1.00 shd ef gr col10 s % Polyline gs clippath 3570 3345 m 3600 3225 l 3630 3345 l 3630 3210 l 3570 3210 l cp 3630 3480 m 3600 3600 l 3570 3480 l 3570 3615 l 3630 3615 l cp clip n 3600 3600 m 3600 3225 l gs col7 1.00 shd ef gr gs col10 s gr gr % arrowhead n 3630 3480 m 3600 3600 l 3570 3480 l 3600 3480 l 3630 3480 l cp gs col10 1.00 shd ef gr col10 s % arrowhead n 3570 3345 m 3600 3225 l 3630 3345 l 3600 3345 l 3570 3345 l cp gs col10 1.00 shd ef gr col10 s % Polyline 0.000 slw n 2850 4575 m 3300 4575 l 3300 4950 l 2850 4950 l cp gs col7 1.00 shd ef gr % Polyline n 5700 4575 m 6150 4575 l 6150 4950 l 5700 4950 l cp gs col7 1.00 shd ef gr % Polyline n 3900 4575 m 4350 4575 l 4350 4950 l 3900 4950 l cp gs col7 1.00 shd ef gr % Polyline n 4650 4575 m 5100 4575 l 5100 4950 l 4650 4950 l cp gs col7 1.00 shd ef gr % Polyline n 1050 4575 m 1500 4575 l 1500 4950 l 1050 4950 l cp gs col7 1.00 shd ef gr % Polyline n 5175 4575 m 5625 4575 l 5625 4950 l 5175 4950 l cp gs col7 1.00 shd ef gr % Polyline n 1575 4575 m 2025 4575 l 2025 4950 l 1575 4950 l cp gs col7 1.00 shd ef gr % Polyline n 3375 4575 m 3825 4575 l 3825 4950 l 3375 4950 l cp gs col7 1.00 shd ef gr % Polyline n 2100 4575 m 2550 4575 l 2550 4950 l 2100 4950 l cp gs col7 1.00 shd ef gr 7.500 slw % Ellipse n 2325 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 2025 4575 m 2025 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 1575 4575 m 1575 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 6150 4575 m 6150 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 5700 4575 m 5700 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 5925 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 5625 4575 m 5625 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 1800 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 5175 4575 m 5175 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 5400 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 5100 4575 m 5100 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 4650 4575 m 4650 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 4875 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 4350 4575 m 4350 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 3900 4575 m 3900 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 4125 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 3825 4575 m 3825 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 3375 4575 m 3375 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 3600 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 3300 4575 m 3300 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 2850 4575 m 2850 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 3075 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 1500 4575 m 1500 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 1050 4575 m 1050 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Ellipse n 1275 4575 225 75 0 360 DrawEllipse gs col-1 s gr % Polyline n 2550 4575 m 2550 4950 l gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 2100 4575 m 2100 4950 l gs col7 1.00 shd ef gr gs col-1 s gr /Times-Bold-iso ff 210.00 scf sf 3975 2025 m gs 1 -1 sc (MPI-IO) dup sw pop 2 div neg 0 rm col10 sh gr $F2psEnd rs %%EndDocument @endspecial -14 1613 a Fe(Figure)25 b(3.)359 1513 y SDict begin H.S end 359 1513 a 359 1513 a SDict begin H.R end 359 1513 a 359 1513 a SDict begin [ /View [/XYZ H.V] /Dest (figure.3) cvn H.B /DEST pdfmark end 359 1513 a 100 x Fe(Design)g(of)g(parallel)h(netCDF)f(on)f(a)i(par)n(\255)-14 1712 y(allel)c(I/O)f(ar)n(c)o(hitecture)r(.)32 b(P)n(arallel)23 b(netCDF)d(runs)i(as)-14 1812 y(a)i(librar)q(y)i(between)d(user)i (space)g(and)f(\002le)h(system)-14 1912 y(space)r(.)31 b(It)22 b(pr)n(ocesses)g(parallel)g(netCDF)f(requests)-14 2011 y(fr)n(om)c(user)h(compute)e(nodes)h(and,)h(after)g(optimiza\255) -14 2111 y(tion,)i(passes)i(the)e(parallel)i(I/O)e(requests)h(do)o(wn)f (to)-14 2210 y(MPI\255IO)38 b(librar)q(y)-7 b(,)44 b(and)39 b(then)f(the)g(I/O)h(ser)q(ver)o(s)h(re\255)-14 2310 y(ceive)j(the)g(MPI\255IO)g(requests)g(and)f(perf)n(orm)h(I/O)-14 2410 y(o)n(ver)23 b(the)g(end)g(stora)o(g)q(e)g(on)f(behalf)h(of)f(the) h(user)-5 b(.)-114 2809 y Fg(cess)25 b(APIs:)34 b(a)24 b Fh(high-le)o(vel)f(API)h Fg(that)g(mimics)g(the)g(serial)h(netCDF) -114 2909 y(data)j(access)h(functions)e(and)g(serv)o(es)i(an)f(easy)g (path)g(for)f(original)-114 3009 y(netCDF)37 b(users)f(to)g(migrate)g (to)g(the)g(parallel)g(interf)o(ace,)j(and)d(a)-114 3108 y Fh(\003e)n(xible)24 b(API)h Fg(that)f(pro)o(vides)f(a)i(more)f (MPI-lik)o(e)h(style)g(of)f(access.)-114 3208 y(Speci\002cally)-5 b(,)15 b(the)h(\003e)o(xible)e(API)i(uses)g(more)e(MPI)i(functionality) d(in)-114 3308 y(order)19 b(to)i(pro)o(vide)d(better)i(handling)e(of)i (internal)g(data)g(representa-)-114 3407 y(tions)e(and)g(to)h(more)f (fully)f(e)o(xpose)h(the)g(capabilities)g(of)g(MPI-IO)g(to)-114 3507 y(the)j(application)f(programmer)-5 b(.)25 b(The)c(major)g(dif)n (ference)e(between)-114 3606 y(the)28 b(tw)o(o)g(is)h(the)f(use)g(of)g (MPI)g(deri)n(v)o(ed)e(data)i(types.)48 b(W)-7 b(e)29 b(belie)n(v)o(e)-114 3706 y(using)21 b(MPI)g(deri)n(v)o(ed)e(datatypes) i(can)g(better)f(illustrate)i(the)f(access)-114 3806 y(patterns)j(than)g(the)h(subarray)e(mapping)g(methods)h(used)g(in)h (origi-)-114 3905 y(nal)20 b(API.)-14 4030 y(The)33 b(most)g(important) e(change)h(from)h(the)g(original)f(netCDF)-114 4129 y(interf)o(ace)d (with)h(respect)f(to)h(data)g(access)g(functions)e(is)j(the)f(split) -114 4229 y(of)c(data)f(mode)g(into)h(tw)o(o)g(distinct)g(modes:)35 b(collecti)n(v)o(e)25 b(and)h(non-)-114 4329 y(collecti)n(v)o(e)k(data) g(modes)g(in)h(which)f(collecti)n(v)o(e)g(function)f(names)-114 4428 y(end)20 b(with)g(\223)p 237 4428 25 4 v 30 w(all\224.)25 b(Similar)c(to)f(MPI-IO,)f(the)i(collecti)n(v)o(e)e(functions)-114 4528 y(are)34 b(synchronous)c(across)k(the)g(processes)f(in)h(the)g (communica-)-114 4628 y(tor)e(associated)h(to)g(the)f(opened)f(netCDF)i (\002le,)j(while)d(the)g(non-)-114 4727 y(collecti)n(v)o(e)h(functions) g(are)h(not.)69 b(Using)35 b(collecti)n(v)o(e)f(operations)-114 4827 y(can)g(pro)o(vide)e(the)i(underlying)e(parallel)h(netCDF)i (implementa-)-114 4926 y(tion)16 b(an)g(opportunity)e(to)i(further)f (optimize)g(access)i(to)g(the)f(netCDF)-114 5026 y(\002le.)33 b(These)23 b(optimizations)e(are)h(performed)e(without)i(further)g(in-) -114 5126 y(terv)o(ention)32 b(by)h(the)g(application)f(programmer)f (and)i(ha)n(v)o(e)g(been)-114 5225 y(pro)o(v)o(en)f(to)i(pro)o(vide)e (dramatic)i(performance)d(impro)o(v)o(ement)g(in)-114 5325 y(multidimensional)18 b(dataset)j(access)g([)1001 5326 y SDict begin H.S end 1001 5326 a -1 x Fg(15)1083 5268 y SDict begin H.R end 1083 5268 a 1083 5325 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.ThGL99) cvn H.B /ANN pdfmark end 1083 5325 a Fg(].)26 b(Figure)1395 5325 y SDict begin H.S end 1395 5325 a Fg(4)1437 5268 y SDict begin H.R end 1437 5268 a 1437 5325 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.4) cvn H.B /ANN pdfmark end 1437 5325 a 21 w Fg(gi)n(v)o(es)20 b(an)g(e)o(x-)2099 1415 y @beginspecial 0 @llx 0 @lly 290 @urx 221 @ury 2227 @rwi @setspecial %%BeginDocument: code.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def /col32 {0.776 0.784 0.784 srgb} bind def /col33 {0.776 0.784 0.784 srgb} bind def end save -4.0 224.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 4729 m -1000 -1000 l 5893 -1000 l 5893 4729 l cp clip 0.06000 0.06000 sc /Times-Roman ff 180.00 scf sf 127 441 m gs 1 -1 sc (1) col0 sh gr /Times-Roman ff 180.00 scf sf 4140 428 m gs 1 -1 sc (&file_id\);) col0 sh gr /Times-Roman ff 180.00 scf sf 450 420 m gs 1 -1 sc (ncmpi_create\(mpi_comm, filename, 0, mpi_info,) col0 sh gr /Times-Roman ff 180.00 scf sf 450 653 m gs 1 -1 sc (ncmpi_def_var\(file_id, ...\);) col0 sh gr /Times-Roman ff 180.00 scf sf 450 878 m gs 1 -1 sc (ncmpi_enddef\(file_id\);) col0 sh gr /Times-Roman ff 180.00 scf sf 128 683 m gs 1 -1 sc (2) col0 sh gr /Times-Roman ff 180.00 scf sf 450 1095 m gs 1 -1 sc (ncmpi_put_vara_all\(file_id, var_id,) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 1276 m gs 1 -1 sc (start[], count[],) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 1439 m gs 1 -1 sc (buffer, bufcount,) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 1613 m gs 1 -1 sc (mpi_datatype\);) col0 sh gr /Times-Roman ff 180.00 scf sf 128 1104 m gs 1 -1 sc (3) col0 sh gr /Times-Roman ff 180.00 scf sf 450 1830 m gs 1 -1 sc (ncmpi_close\(file_id\);) col0 sh gr /Times-Roman ff 180.00 scf sf 120 1853 m gs 1 -1 sc (4) col0 sh gr /Times-Roman ff 180.00 scf sf 127 2415 m gs 1 -1 sc (1) col0 sh gr /Times-Roman ff 180.00 scf sf 450 2401 m gs 1 -1 sc (ncmpi_open\(mpi_comm, filename, 0, mpi_info,) col0 sh gr /Times-Roman ff 180.00 scf sf 4088 2401 m gs 1 -1 sc (&file_id\);) col0 sh gr /Times-Bold ff 180.00 scf sf 74 2190 m gs 1 -1 sc (\(b\) READ:) col0 sh gr /Times-Roman ff 180.00 scf sf 128 2677 m gs 1 -1 sc (2) col0 sh gr /Times-Roman ff 180.00 scf sf 450 2655 m gs 1 -1 sc (ncmpi_inq\(file_id, ... \);) col0 sh gr /Times-Roman ff 180.00 scf sf 128 2910 m gs 1 -1 sc (3) col0 sh gr /Times-Roman ff 180.00 scf sf 450 2895 m gs 1 -1 sc (ncmpi_get_vars_all\(file_id, var_id,) col0 sh gr /Times-Roman ff 180.00 scf sf 120 3690 m gs 1 -1 sc (4) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 3270 m gs 1 -1 sc (buffer, bufcount,) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 3082 m gs 1 -1 sc (start[], count[], stride[],) col0 sh gr /Times-Roman ff 180.00 scf sf 450 3675 m gs 1 -1 sc (ncmpi_close\(file_id\);) col0 sh gr /Times-Roman ff 180.00 scf sf 1950 3443 m gs 1 -1 sc (mpi_datatype\);) col0 sh gr /Times-Bold ff 180.00 scf sf 74 231 m gs 1 -1 sc (\(a\) WRITE:) col0 sh gr $F2psEnd rs %%EndDocument @endspecial 2142 1620 a Fe(Figure)34 b(4.)2533 1521 y SDict begin H.S end 2533 1521 a 2533 1521 a SDict begin H.R end 2533 1521 a 2533 1521 a SDict begin [ /View [/XYZ H.V] /Dest (figure.4) cvn H.B /DEST pdfmark end 2533 1521 a 99 x Fe(Example)h(of)e(using)h(parallel)g(netCDF)-8 b(.)2142 1720 y(T)j(ypicall)o(y)44 b(there)g(are)h(4)g(main)f(steps:)74 b(1.)96 b(col\255)2142 1820 y(lectivel)o(y)34 b(create/open)f(the)g (dataset;)38 b(2.)63 b(collec\255)2142 1919 y(tivel)o(y)43 b(de\002ne)g(the)g(dataset)g(b)n(y)h(ad)o(ding)e(dimen\255)2142 2019 y(sions,)35 b(v)n(ariab)o(les)g(and)d(attrib)n(utes)h(in)g(WRITE,) g(or)2142 2118 y(inquir)q(y)23 b(about)f(the)h(dataset)g(to)f(g)q(et)h (metadata)f(as\255)2142 2218 y(sociated)g(with)g(the)g(dataset)h(in)f (READ;)h(3.)32 b(access)2142 2318 y(the)24 b(data)h(arra)n(ys)h (\(collective)f(or)g(non\255collective\);)2142 2417 y(4.)32 b(collectivel)o(y)24 b(c)n(lose)g(the)e(dataset.)2042 2776 y Fg(ample)i(code)g(of)g(using)f(our)h(parallel)g(netCDF)h(API)f (to)h(write)f(and)2042 2876 y(read)c(a)h(dataset)f(using)f(collecti)n (v)o(e)h(I/O.)2042 3007 y SDict begin H.S end 2042 3007 a 2042 3007 a SDict begin 12 H.A end 2042 3007 a 2042 3007 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.4.2) cvn H.B /DEST pdfmark end 2042 3007 a 86 x Ff(4.2)91 b(P)o(arallel)22 b(Implementation)2142 3310 y Fg(The)38 b(parallel)g(API)h (implementation)e(is)i(discussed)g(in)f(tw)o(o)2042 3409 y(parts:)47 b(header)30 b(I/O)h(and)f(parallel)h(data)g(I/O.)g(W)-7 b(e)32 b(\002rst)f(describe)2042 3509 y(out)c(implementation)e(strate)o (gies)j(for)e(dataset)i(functions,)f(de\002ne)2042 3608 y(mode)h(functions,)i(attrib)n(ute)e(functions,)h(and)g(inquiry)e (functions)2042 3708 y(that)21 b(access)f(the)h(netCDF)f(\002le)h (header)-5 b(.)2042 3813 y SDict begin H.S end 2042 3813 a 2042 3813 a SDict begin 12 H.A end 2042 3813 a 2042 3813 a SDict begin [ /View [/XYZ H.V] /Dest (subsubsection.4.2.1) cvn H.B /DEST pdfmark end 2042 3813 a 103 x Fd(4.2.1)81 b(Access)21 b(to)f(File)h(Header)2142 4125 y Fg(Internally)-5 b(,)36 b(the)f(header)e(is)j(read/written)d(only)h(by)h(a)g(single)2042 4225 y(process,)40 b(although)35 b(a)i(cop)o(y)f(is)i(cached)e(in)g (local)h(memory)e(on)2042 4324 y(each)c(process.)57 b(The)31 b(de\002ne)f(mode)g(functions,)i(attrib)n(ute)f(func-)2042 4424 y(tions,)f(and)e(inquiry)e(functions)h(all)i(w)o(ork)e(on)h(the)g (local)h(cop)o(y)e(of)2042 4523 y(the)20 b(\002le)h(header)-5 b(.)25 b(Since)20 b(the)o(y)f(are)i(all)f(in-memory)e(operations)h(not) 2042 4623 y(in)m(v)n(olv)o(ed)30 b(in)i(an)o(y)g(\002le)g(I/O,)h(the)o (y)e(bear)g(fe)n(w)h(changes)g(from)f(the)2042 4723 y(serial)25 b(netCDF)g(API.)g(The)o(y)e(are)i(made)f(collecti)n(v)o(e,)g(b)n(ut)h (this)g(fea-)2042 4822 y(ture)h(does)h(not)f(necessarily)g(imply)g (inter)n(-process)f(synchroniza-)2042 4922 y(tion.)j(In)21 b(some)g(cases,)h(ho)n(we)n(v)o(er)m(,)d(when)i(the)g(header)f (de\002nition)h(is)2042 5022 y(changed)f(synchronization)e(is)k(needed) e(to)h(v)o(erify)f(that)h(the)g(v)n(alues)2042 5121 y(passed)26 b(in)g(by)f(all)h(processes)g(match.)41 b(In)25 b(all)h(possible)g (cases)g(we)2042 5221 y(allo)n(w)20 b(inter)n(-process)f (communications.)2142 5325 y(The)29 b(dataset)g(functions,)g(unlik)o(e) g(the)g(other)f(functions)g(cited,)1928 5574 y(5)p eop %%Page: 6 6 6 5 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.6) cvn H.B /DEST pdfmark end -114 0 a 83 x Fg(need)57 b(complete)g(reimplementation)f (because)h(the)o(y)g(are)h(in)-114 183 y(char)o(ge)29 b(of)i(collecti)n(v)o(ely)e(opening/creating)e(datasets,)34 b(perform-)-114 282 y(ing)d(header)f(I/O)i(and)f(\002le)h (synchronization)c(for)j(all)h(processes,)-114 382 y(and)16 b(managing)e(inter)n(-process)h(communication.)21 b(W)-7 b(e)17 b(b)n(uild)f(these)-114 482 y(functions)33 b(o)o(v)o(er)h (MPI-IO)g(so)h(that)g(the)o(y)f(ha)n(v)o(e)g(better)h(portabil-)-114 581 y(ity)29 b(and)f(pro)o(vide)e(more)i(optimization)e(opportunities.) 48 b(The)28 b(ba-)-114 681 y(sic)k(idea)g(is)g(to)g(let)g(the)g(R)m(OO) m(T)f(process)g(fetch)g(the)h(\002le)g(header)m(,)-114 780 y(broadcast)22 b(it)i(to)f(all)g(processes)g(when)g(opening)e(a)i (\002le,)i(and)d(write)-114 880 y(the)38 b(\002le)g(header)f(at)h(the)g (end)f(of)h(de\002nition)e(if)j(an)o(y)e(modi\002ca-)-114 980 y(tion)28 b(occurs)g(in)g(the)h(header)e(part.)49 b(Since)29 b(all)g(de\002ne)e(mode)h(and)-114 1079 y(attrib)n(ute)h (functions)e(are)i(collecti)n(v)o(e)g(and)f(require)g(all)i(processes) -114 1179 y(in)c(the)g(communicator)c(to)k(pro)o(vide)e(the)h(same)h (ar)o(guments)e(when)-114 1279 y(adding/remo)o(ving/chang)o(ing)h (de\002nitions,)32 b(the)f(local)g(copies)g(of)-114 1378 y(the)26 b(\002le)h(header)e(shall)i(be)f(the)g(same)g(across)g(all)h (processes)f(once)-114 1478 y(the)20 b(\002le)h(is)g(collecti)n(v)o (ely)e(opened)g(and)g(until)h(it)h(is)g(closed.)-114 1604 y SDict begin H.S end -114 1604 a -114 1604 a SDict begin 12 H.A end -114 1604 a -114 1604 a SDict begin [ /View [/XYZ H.V] /Dest (subsubsection.4.2.2) cvn H.B /DEST pdfmark end -114 1604 a 91 x Fd(4.2.2)81 b(P)o(arallel)20 b(I/O)g(f)n(or)g(Array) g(Data)-14 1911 y Fg(Since)g(the)g(majority)f(of)h(time)g(spent)g (accessing)g(a)g(netCDF)h(\002le)-114 2011 y(is)k(in)f(data)g(access,)h (the)f(data)g(I/O)g(must)g(be)g(ef)n(\002cient.)36 b(By)24 b(imple-)-114 2111 y(menting)16 b(the)h(data)h(access)f(functions)f (abo)o(v)o(e)g(MPI-IO,)g(we)i(enable)-114 2210 y(a)j(number)d(of)i(adv) n(antages)e(and)i(optimizations.)-14 2319 y(F)o(or)25 b(each)h(of)g(the)g(\002)n(v)o(e)f(data)h(access)h(methods)e(in)h(the)g (\003e)o(xible)-114 2418 y(data)f(access)g(functions,)g(we)g(represent) f(the)h(data)g(access)g(pattern)-114 2518 y(as)33 b(an)g(MPI)g(\002le)g (vie)n(w)g(\(a)g(set)g(of)g(data)f(visible)h(and)f(accessible)-114 2618 y(from)24 b(an)h(open)f(\002le)i([)524 2619 y SDict begin H.S end 524 2619 a -1 x Fg(7)565 2562 y SDict begin H.R end 565 2562 a 565 2618 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.Mess97) cvn H.B /ANN pdfmark end 565 2618 a Fg(]\),)g(which)f(is)h(constructed)d(from)h(the)h(v)n(ari-)-114 2717 y(able)g(metadata)g(\(shape,)h(size,)i(of)n(fset,)e(etc.\))41 b(in)26 b(the)g(netCDF)g(\002le)-114 2817 y(header)31 b(and)g(start[],)j(count[],)e(stride[],)i(imap[],)f(mpi)p 1549 2817 25 4 v 29 w(datatype)-114 2916 y(ar)o(guments)24 b(pro)o(vided)f(by)i(users.)42 b(F)o(or)26 b(parallel)f(access,)j (particu-)-114 3016 y(larly)c(for)h(collecti)n(v)o(e)e(access,)k(each)d (process)g(has)h(a)h(dif)n(ferent)d(\002le)-114 3116 y(vie)n(w;)31 b(and)c(all)h(processes)f(in)h(combination)d(can)i(mak)o (e)g(a)h(single)-114 3215 y(MPI-IO)16 b(request)f(to)i(transfer)e(lar)o (ge)g(contiguous)f(data)i(as)h(a)g(whole,)-114 3315 y(thereby)23 b(preserving)f(useful)h(semantic)h(information)e(that)i(w)o(ould)-114 3415 y(otherwise)f(be)g(lost)h(if)g(the)g(transfer)e(were)i(e)o (xpressed)e(as)i(per)f(pro-)-114 3514 y(cess)e(noncontiguous)c (requests.)-14 3623 y(The)34 b(high-le)n(v)o(el)f(data)i(access)g (functions)e(are)i(implemented)-114 3722 y(in)c(terms)g(of)g(the)g (\003e)o(xible)f(data)h(access)h(functions,)g(so)f(that)g(e)o(x-)-114 3822 y(isting)g(users)g(migrating)f(from)g(serial)i(netCDF)f(can)g (also)g(bene-)-114 3921 y(\002t)f(from)f(the)g(MPI-IO)g(optimizations.) 51 b(Ho)n(we)n(v)o(er)m(,)30 b(the)f(\003e)o(xible)-114 4021 y(data)21 b(access)h(functions)e(are)i(closer)f(to)h(MPI-IO)f(and) g(hence)f(incur)-114 4121 y(less)32 b(o)o(v)o(erhead.)56 b(The)o(y)30 b(accept)h(a)h(user)n(-speci\002ed)f(MPI)g(deri)n(v)o(ed) -114 4220 y(datatype)16 b(and)g(pass)h(it)h(directly)e(to)h(MPI-IO)f (for)g(optimal)g(handling)-114 4320 y(of)k(in-memory)e(data)i(access)g (patterns.)-14 4428 y(In)30 b(some)g(cases)h(\(for)f(instance,)i(in)f (record)e(v)n(ariable)g(access\))-114 4528 y(the)e(data)h(is)g(stored)f (interlea)n(v)o(ed)f(by)h(record)f(and)h(the)g(contiguity)-114 4628 y(information)h(is)j(lost,)j(so)d(the)f(e)o(xisting)g(MPI-IO)g (collecti)n(v)o(e)f(I/O)-114 4727 y(optimization)17 b(may)h(not)g (help.)24 b(In)19 b(that)f(case,)h(we)g(need)f(more)g(opti-)-114 4827 y(mization)f(information)e(from)i(users,)h(such)f(as)i(the)e (number)m(,)f(order)m(,)-114 4926 y(and)25 b(record)g(indices)h(of)f (the)h(record)f(v)n(ariables)g(the)o(y)g(will)i(access)-114 5026 y(consecuti)n(v)o(ely)-5 b(.)25 b(W)m(ith)c(such)f(information)f (we)i(can)g(collect)g(multi-)-114 5126 y(ple)e(I/O)g(requests)f(o)o(v)o (er)g(a)h(number)e(of)h(record)g(v)n(ariables)g(and)g(opti-)-114 5225 y(mize)i(the)h(\002le)g(I/O)g(o)o(v)o(er)e(a)i(lar)o(ge)f(pool)f (of)i(data)f(transfers,)g(thereby)-114 5325 y(producing)h(more)i (contiguous)f(and)h(lar)o(ger)g(transfers.)35 b(This)24 b(kind)2042 83 y(of)19 b(information)e(is)j(passed)g(in)f(as)h(an)f (MPI)p 3284 83 V 30 w(Info)f(hint)h(when)g(a)h(user)2042 183 y(opens)h(or)h(creates)g(a)h(netCDF)f(dataset.)31 b(W)-7 b(e)23 b(implement)d(our)i(user)2042 282 y(hints)28 b(in)g(parallel)f(netCDF)h(for)f(all)h(such)f(speci\002c)h (optimization)2042 382 y(points,)h(while)e(a)h(number)d(of)i(standard)g (hints)g(are)g(passed)h(do)n(wn)2042 482 y(for)h(MPI-IO)f(to)h(tak)o(e) g(control)e(of)i(optimal)f(parallel)h(I/O)g(beha)n(v-)2042 581 y(iors.)45 b(Thus)26 b(e)o(xperienced)e(users)j(ha)n(v)o(e)g(the)f (opportunity)e(to)j(tune)2042 681 y(their)20 b(applications)f(for)h (further)e(performance)g(gains.)2042 814 y SDict begin H.S end 2042 814 a 2042 814 a SDict begin 12 H.A end 2042 814 a 2042 814 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.4.3) cvn H.B /DEST pdfmark end 2042 814 a 89 x Ff(4.3)91 b(Adv)o(antages)20 b(and)i(Disadv)o(antages)2142 1126 y Fg(There)38 b(are)h(a)h(number)d(of)i(adv)n(antages)e(within)i(the)g (design)2042 1225 y(and)31 b(implementation)f(of)h(our)g(parallel)g (netCDF)-7 b(,)32 b(as)h(compared)2042 1325 y(to)21 b(other)e(related)h (w)o(ork,)f(lik)o(e)h(HDF5.)2142 1432 y(First)i(of)f(all,)h(the)f (parallel)f(netCDF)i(design)f(and)f(implementa-)2042 1532 y(tion)g(is)h(optimized)d(for)i(the)g(netCDF)g(\002le)h(format)d (so)j(that)f(the)g(data)2042 1631 y(I/O)31 b(performance)d(is)k(as)g (good)d(as)j(the)f(MPI-IO.)f(The)h(NetCDF)2042 1731 y(\002le)24 b(chooses)e(linear)g(data)g(layout,)h(in)f(which)g(the)h(data)g(arrays) f(are)2042 1831 y(either)h(stored)g(in)h(contiguous)d(space)j(and)f(in) g(a)h(prede\002ned)e(order)2042 1930 y(or)i(interlea)n(v)o(ed)e(in)j(a) f(re)o(gular)e(pattern.)36 b(This)24 b(re)o(gular)e(and)i(highly)2042 2030 y(predictable)34 b(data)i(layout)f(enables)g(the)g(parallel)h (netCDF)g(data)2042 2129 y(I/O)18 b(implementation)e(to)i(simply)f (pass)h(the)g(data)g(b)n(uf)n(fer)m(,)e(metadata)2042 2229 y(\(\002le)n(vie)n(w)-5 b(,)22 b(mpi)p 2507 2229 25 4 v 29 w(datatype,)f(etc.\),)h(and)g(other)f(optimization)g(infor)n (-)2042 2329 y(mation)27 b(to)h(MPI-IO,)e(and)h(all)h(parallel)f(I/O)h (operations)e(are)i(car)n(-)2042 2428 y(ried)20 b(out)g(in)g(the)g (same)h(manner)d(as)j(when)f(MPI-IO)f(alone)h(is)h(used.)2042 2528 y(Thus,)h(there)g(is)h(v)o(ery)e(little)i(o)o(v)o(erhead,)d(and)i (the)g(parallel)g(netCDF)2042 2628 y(performance)h(should)i(be)h (nearly)g(the)g(same)g(as)h(MPI-IO)e(if)h(only)2042 2727 y(ra)o(w)d(data)f(I/O)h(performance)c(is)24 b(compared.)30 b(On)23 b(the)f(other)g(hand,)2042 2827 y(parallel)29 b(HDF5)h(uses)g(tree-lik)o(e)f(\002le)h(structure)e(that)i(are)f (similar)2042 2926 y(to)34 b(the)f(UNIX)h(\002le)g(system)g(and)f(the)g (data)h(is)g(dispersedly)e(laid)2042 3026 y(out)20 b(using)f(super)h (block,)f(header)f(blocks,)i(data)f(blocks,)h(e)o(xtended)2042 3126 y(header)g(blocks)g(and)g(e)o(xtended)e(data)j(blocks.)k(This)c (irre)o(gular)e(lay-)2042 3225 y(out)i(pattern)f(may)g(mak)o(e)h(it)g (dif)n(\002cult)g(to)g(pass)g(user)g(access)g(pattern)2042 3325 y(directly)27 b(to)g(MPI-IO)g(especially)g(for)g(the)g(case)h(of)f (v)n(ariable-size)2042 3425 y(arrays.)50 b(Instead,)30 b(parallel)e(HDF5)g(uses)h(dataspace)f(and)g(hyper)n(-)2042 3524 y(slabs)23 b(to)g(de\002ne)f(the)g(data)g(or)o(ganization,)e(map)h (and)h(transfer)g(data)2042 3624 y(between)k(memory)g(space)h(and)f (the)h(\002le)h(space)f(and)f(does)h(b)n(uf)n(fer)2042 3724 y(packing/unpacking)17 b(in)22 b(a)g(recursi)n(v)o(e)e(w)o(ay)-5 b(,)21 b(while)h(these)g(can)f(oth-)2042 3823 y(erwise)30 b(be)f(directly)g(handled)e(by)i(MPI-IO)g(in)h(a)f(more)g(ef)n (\002cient)2042 3923 y(and)20 b(optimized)f(w)o(ay)-5 b(.)2142 4030 y(Secondly)g(,)22 b(the)h(parallel)g(netCDF)h (implementation)d(manages)2042 4129 y(to)d(k)o(eep)g(the)g(o)o(v)o (erhead)e(in)m(v)n(olv)o(ed)g(in)i(header)f(I/O)h(as)h(lo)n(w)f(as)h (possi-)2042 4229 y(ble.)25 b(In)19 b(the)h(netCDF)f(\002le,)h(there)f (is)i(only)d(one)h(header)g(which)f(con-)2042 4329 y(tains)f(all)h (necessary)e(information)e(for)j(direct)f(access)i(of)e(each)h(data) 2042 4428 y(array)g(and)g(each)h(array)f(is)h(associated)g(with)g(a)g (prede\002ned,)e(numer)n(-)2042 4528 y(ical)32 b(ID)f(that)g(can)g(be)g (ef)n(\002ciently)f(inquired)f(when)h(it)i(is)g(needed)2042 4628 y(to)j(access)g(the)g(array)-5 b(.)66 b(So,)39 b(by)34 b(maintaining)f(a)i(local)f(cop)o(y)g(of)2042 4727 y(the)23 b(header)e(on)h(each)g(process,)h(our)e(implementation)f(sa)n(v)o(es)j (a)g(lot)2042 4827 y(of)g(inter)n(-process)f(synchronization)e(as)k (well)g(as)g(a)n(v)n(oids)f(repeated)2042 4926 y(access)k(of)g(the)f (\002le)i(header)d(each)h(time)h(the)g(header)e(information)2042 5026 y(is)j(needed)d(to)i(access)g(a)g(single)f(array)-5 b(.)42 b(All)28 b(header)d(information)2042 5126 y(can)d(be)g(accessed) g(directly)g(in)g(local)g(memory)e(and)i(inter)n(-process)2042 5225 y(synchronization)f(is)j(needed)f(only)f(during)g(the)i (de\002nition)e(of)i(the)2042 5325 y(dataset.)50 b(And)27 b(once)h(the)g(de\002nition)g(of)g(the)g(dataset)g(is)i(created,)1928 5574 y(6)p eop %%Page: 7 7 7 6 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.7) cvn H.B /DEST pdfmark end -114 0 a 83 x Fg(each)31 b(array)f(can)h(be)g(identi\002ed)g (by)g(its)h(permanent)d(ID)j(and)e(ac-)-114 183 y(cessed)f(at)g(an)o(y) e(time)i(by)f(an)o(y)f(process,)j(without)e(an)o(y)f(collecti)n(v)o(e) -114 282 y(open/close)20 b(operation.)25 b(Ho)n(we)n(v)o(er)m(,)19 b(in)j(HDF5,)f(the)g(header)f(meta-)-114 382 y(data)25 b(is)i(dispersed)e(in)g(separate)g(header)g(blocks)g(for)g(each)g (object)-114 482 y(and,)17 b(in)f(order)g(to)g(operate)g(on)g(an)g (object,)h(it)g(has)g(to)g(iterate)f(through)-114 581 y(the)26 b(entire)g(namespace)f(to)h(get)g(the)g(header)f(information)e (of)j(that)-114 681 y(object)e(and)f(then)h(open,)g(access)g(and)g (close)g(it.)38 b(This)24 b(kind)g(of)f(ac-)-114 780 y(cess)d(method)d(may)i(be)f(inef)n(\002cient)g(for)g(parallel)h (access,)g(since)g(the)-114 880 y(parallel)g(HDF5)h(designs)g(the)g (open/close)e(of)i(each)g(object)f(as)i(col-)-114 980 y(lecti)n(v)o(e)30 b(operations,)i(which)e(force)g(all)h(participating) e(processes)-114 1079 y(to)h(communicate)e(when)h(accessing)h(one)f (single)h(object,)i(not)d(to)-114 1179 y(mention)17 b(the)g(cost)i(of)e (\002le)i(access)f(to)g(locate)g(and)f(fetch)h(the)f(header)-114 1279 y(information)h(of)i(that)g(object.)-14 1423 y(Lastly)-5 b(,)16 b(the)h(programming)c(interf)o(ace)j(of)g(the)h(parallel)f (netCDF)-114 1522 y(is)i(concise)e(and)g(designed)g(for)g(easy)h (usage,)f(and)h(the)f(\002le)i(format)d(is)-114 1622 y(fully)21 b(compatible)e(with)j(serial)f(netCDF)-7 b(.)22 b(Porting)e(e)o(xisting)h(serial)-114 1721 y(netCDF)26 b(application)e(to)i(parallel)g(netCDF)g(should)e(be)i(straight-)-114 1821 y(forw)o(ard)f(because)h(the)h(parallel)f(API)h(contains)f(nearly) f(all)j(func-)-114 1921 y(tions)18 b(of)h(the)f(serial)h(API)g(with)f (parallel)g(semantics)h(b)n(ut)f(with)h(min-)-114 2020 y(imum)h(change)e(of)i(function)f(names)h(and)f(ar)o(gument)f(lists.) -14 2164 y(Ho)n(we)n(v)o(er)m(,)28 b(there)g(are)h(also)g(limitations)f (in)h(parallel)f(netCDF)-7 b(.)-114 2264 y(Unlik)o(e)22 b(HDF5,)g(netCDF)h(does)f(not)g(support)e(hierarchical)h(group)-114 2364 y(based)j(or)o(ganization)c(of)k(data)g(objects)g(and)f(since)h (it)h(lays)g(out)e(the)-114 2463 y(data)i(in)h(a)f(linear)g(order)m(,)g (adding)f(\002x)o(ed-size)g(array)h(or)g(e)o(xtending)-114 2563 y(the)j(\002le)h(header)e(may)g(be)h(v)o(ery)f(costly)h(once)f (the)h(\002le)h(is)g(created)-114 2662 y(and)j(has)h(e)o(xisting)f (data)g(stored,)j(though)c(mo)o(ving)f(the)j(e)o(xisting)-114 2762 y(data)17 b(to)g(e)o(xtended)f(area)g(is)j(performed)14 b(in)j(parallel.)24 b(Also,)18 b(parallel)-114 2862 y(netCDF)28 b(does)g(not)f(pro)o(vide)f(functionality)g(to)i(combine)e(tw)o(o)j(or) -114 2961 y(more)19 b(\002les)h(in)f(memory)f(through)f(softw)o(are)i (mounting,)e(as)j(HDF5)-114 3061 y(does.)k(Nor)17 b(does)h(netCDF)g (support)f(data)h(compression)e(within)h(its)-114 3161 y(\002le)26 b(format.)39 b(F)o(ortunately)-5 b(,)23 b(these)j(features) e(can)h(all)h(be)f(achie)n(v)o(ed)-114 3260 y(by)c(e)o(xternal)e(softw) o(are,)i(sacri\002cing)g(some)f(manageability)g(of)g(the)-114 3360 y(\002les.)-114 3521 y SDict begin H.S end -114 3521 a -114 3521 a SDict begin 12 H.A end -114 3521 a -114 3521 a SDict begin [ /View [/XYZ H.V] /Dest (section.5) cvn H.B /DEST pdfmark end -114 3521 a 160 x Fi(5)99 b(P)n(erf)n(ormance)26 b(Ev)o(aluation)-14 3986 y Fg(T)-7 b(o)26 b(e)n(v)n(aluate)g(the)h (performance)c(and)j(scalability)h(of)f(our)g(par)n(-)-114 4085 y(allel)i(netCDF)g(with)g(that)g(of)f(serial)h(netCDF)-7 b(,)28 b(we)g(ran)g(some)f(e)o(x-)-114 4185 y(periments)20 b(and)h(compared)f(the)h(results.)29 b(W)-7 b(e)23 b(also)f(compared)d (the)-114 4284 y(performance)c(of)j(parallel)g(netCDF)g(with)h(that)f (of)g(parallel)f(HDF5,)-114 4384 y(using)j(the)g(FLASH)h(I/O)f (benchmark.)-14 4528 y(The)34 b(e)o(xperiments)f(were)h(run)g(on)g(an)h (IBM)g(SP-2)g(machine.)-114 4628 y(This)15 b(system)h(is)g(a)g (tera\003op-scale)e(clustered)g(SMP)i(with)g(144)e(com-)-114 4727 y(pute)h(nodes.)23 b(Each)14 b(compute)g(node)h(has)g(4)h(GB)g(of) f(memory)f(shared)-114 4827 y(among)20 b(its)i(eight)f(375)f(MHz)h(Po)n (wer3)f(processors.)27 b(All)22 b(the)f(com-)-114 4926 y(pute)33 b(nodes)g(are)g(interconnected)e(by)i(switches)h(and)e(also)i (con-)-114 5026 y(nected)27 b(via)h(switches)h(to)f(the)g(multiple)g (I/O)g(nodes)f(running)f(the)-114 5126 y(GPFS)32 b(parallel)e(\002le)h (system.)56 b(There)30 b(are)g(12)h(I/O)f(nodes,)j(each)-114 5225 y(with)23 b(dual)e(222)h(MHz)g(processes.)32 b(The)22 b(aggre)o(gate)e(disk)i(space)g(is)-114 5325 y(5)e(TB)h(and)f(the)g (peak)f(I/O)i(bandwidth)d(is)j(1.5)f(GB/s.)2042 2120 y @beginspecial 0 @llx 0 @lly 497 @urx 530 @ury 2376 @rwi @setspecial %%BeginDocument: partition.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def end save -18.0 549.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 10150 m -1000 -1000 l 9572 -1000 l 9572 10150 l cp clip 0.06000 0.06000 sc % Polyline 15.000 slw n 600 1050 m 675 975 l 2475 975 l 2475 2775 l 2400 2850 l 2400 1050 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 750 900 m 825 825 l 2625 825 l 2625 2625 l 2550 2700 l 2550 900 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 900 750 m 975 675 l 2775 675 l 2775 2475 l 2700 2550 l 2700 750 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 975 675 m 1050 600 l 2850 600 l 2850 2400 l 2775 2475 l 2775 675 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 1050 600 m 1125 525 l 2925 525 l 2925 2325 l 2850 2400 l 2850 600 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 1125 525 m 1200 450 l 3000 450 l 3000 2250 l 2925 2325 l 2925 525 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 825 825 m 900 750 l 2700 750 l 2700 2550 l 2625 2625 l 2625 825 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 4725 3600 m 4875 3450 l 5775 3450 l 5775 5250 l 5625 5400 l 5625 3600 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 4275 4050 m 4425 3900 l 5325 3900 l 5325 5700 l 5175 5850 l 4275 5850 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 1050 3600 m 1200 3450 l 3000 3450 l 3000 4350 l 2850 4500 l 2850 3600 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 675 975 m 750 900 l 2550 900 l 2550 2700 l 2475 2775 l 2475 975 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline 30.000 slw gs clippath 751 632 m 873 551 l 793 674 l 953 514 l 911 472 l cp clip n 450 975 m 900 525 l gs col0 s gr gr % arrowhead 15.000 slw n 751 632 m 873 551 l 793 674 l 789 636 l 751 632 l cp gs 0.00 setgray ef gr col0 s /Times-Bold ff 270.00 scf sf 825 525 m gs 1 -1 sc (Z) col0 sh gr % Polyline n 600 4050 m 750 3900 l 2550 3900 l 2550 4800 l 2400 4950 l 600 4950 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 600 4950 m 2400 4950 l 2550 4800 l 2550 5700 l 2400 5850 l 600 5850 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 3375 1275 m 5175 1275 l 5775 675 l 5775 900 l 5175 1500 l 3375 1500 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 3375 1500 m 5175 1500 l 5775 900 l 5775 1125 l 5175 1725 l 3375 1725 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 3375 1725 m 5175 1725 l 5775 1125 l 5775 1350 l 5175 1950 l 3375 1950 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 3375 1950 m 5175 1950 l 5775 1350 l 5775 1575 l 5175 2175 l 3375 2175 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 3375 2175 m 5175 2175 l 5775 1575 l 5775 1800 l 5175 2400 l 3375 2400 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 3375 2400 m 5175 2400 l 5775 1800 l 5775 2025 l 5175 2625 l 3375 2625 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 3375 2625 m 5175 2625 l 5775 2025 l 5775 2250 l 5175 2850 l 3375 2850 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 1800 6750 m 2100 6450 l 3000 6450 l 3000 7350 l 2700 7650 l 2700 6750 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 3375 1050 m 3975 450 l 5775 450 l 5775 675 l 5175 1275 l 3375 1275 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 4575 3750 m 4725 3600 l 5625 3600 l 5625 5400 l 5475 5550 l 5475 3750 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 900 6750 m 1200 6450 l 2100 6450 l 1800 6750 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 6150 1050 m 6750 450 l 6975 450 l 6375 1050 l 6375 2850 l 6150 2850 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 6375 1050 m 6975 450 l 7200 450 l 6600 1050 l 6600 2850 l 6375 2850 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 6600 1050 m 7200 450 l 7425 450 l 6825 1050 l 6825 2850 l 6600 2850 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 6825 1050 m 7425 450 l 7650 450 l 7050 1050 l 7050 2850 l 6825 2850 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 7050 1050 m 7650 450 l 7875 450 l 7275 1050 l 7275 2850 l 7050 2850 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 7275 1050 m 7875 450 l 8100 450 l 7500 1050 l 7500 2850 l 7275 2850 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 7500 1050 m 8100 450 l 8325 450 l 7725 1050 l 7725 2850 l 7500 2850 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 4425 3900 m 4575 3750 l 5475 3750 l 5475 5550 l 5325 5700 l 5325 3900 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 7725 1050 m 8325 450 l 8550 450 l 8550 2250 l 7950 2850 l 7725 2850 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 3375 4050 m 3525 3900 l 4425 3900 l 4275 4050 l 4275 5850 l 3375 5850 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 1500 7950 m 2400 7950 l 2700 7650 l 2700 8550 l 2400 8850 l 1500 8850 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 600 7950 m 1500 7950 l 1500 8850 l 600 8850 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 750 3900 m 900 3750 l 2700 3750 l 2700 4650 l 2550 4800 l 2550 3900 l cp gs 0.75 setgray ef gr gs col0 s gr /Times-Bold ff 270.00 scf sf 300 1950 m gs 1 -1 sc (Y) col0 sh gr % Polyline n 900 3750 m 1050 3600 l 2850 3600 l 2850 4500 l 2700 4650 l 2700 3750 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 7050 4950 m 7950 4950 l 8550 4350 l 8550 4800 l 7950 5400 l 7050 5400 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 6150 4950 m 7050 4950 l 7050 5400 l 6150 5400 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 7050 4500 m 7950 4500 l 8550 3900 l 8550 4350 l 7950 4950 l 7050 4950 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 6150 4500 m 7050 4500 l 7050 4950 l 6150 4950 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 7050 4500 m 7050 4050 l 7650 3450 l 8550 3450 l 8550 3900 l 7950 4500 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 6150 4500 m 6150 4050 l 6750 3450 l 7650 3450 l 7050 4050 l 7050 4500 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 6150 5400 m 7050 5400 l 7050 5850 l 6150 5850 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 7050 5400 m 7950 5400 l 8550 4800 l 8550 5250 l 7950 5850 l 7050 5850 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 600 7950 m 600 7050 l 900 6750 l 1800 6750 l 1500 7050 l 1500 7950 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 1500 7050 m 1800 6750 l 2700 6750 l 2700 7650 l 2400 7950 l 1500 7950 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 3375 1050 m 3975 450 l 5775 450 l 5775 2250 l 5175 2850 l gs col0 s gr % Polyline n 5175 4050 m 5775 3450 l gs col0 s gr % Polyline n 3675 3750 m 3825 3600 l 4725 3600 l 4575 3750 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline n 6150 1050 m 7950 1050 l 7950 2850 l 6150 2850 l cp gs col0 s gr % Polyline n 3825 3600 m 3975 3450 l 4875 3450 l 4725 3600 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 2400 1050 m 3000 450 l gs col0 s gr % Polyline n 3375 4050 m 5175 4050 l 5175 5850 l 3375 5850 l cp gs col0 s gr % Polyline n 7950 4050 m 8550 3450 l gs col0 s gr % Polyline n 6150 4050 m 7950 4050 l 7950 5850 l 6150 5850 l cp gs col0 s gr % Polyline n 5175 1050 m 5775 450 l gs col0 s gr % Polyline n 3375 1050 m 5175 1050 l 5175 2850 l 3375 2850 l cp gs col0 s gr % Polyline n 3525 3900 m 3675 3750 l 4575 3750 l 4425 3900 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 2850 4500 m 3000 4350 l 3000 5250 l 2850 5400 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 600 1050 m 2400 1050 l 2400 2850 l 600 2850 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 6150 1050 m 6750 450 l 8550 450 l 8550 2250 l 7950 2850 l gs col0 s gr % Polyline n 6150 4050 m 6750 3450 l 8550 3450 l 8550 5250 l 7950 5850 l gs col0 s gr % Polyline n 600 4050 m 2400 4050 l 2400 5850 l 600 5850 l cp gs col0 s gr % Polyline n 600 7050 m 1200 6450 l 3000 6450 l 3000 8250 l 2400 8850 l gs col0 s gr % Polyline n 2400 7050 m 3000 6450 l gs col0 s gr % Polyline n 2550 4800 m 2700 4650 l 2700 5550 l 2550 5700 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 2700 7650 m 3000 7350 l 3000 8250 l 2700 8550 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 2700 4650 m 2850 4500 l 2850 5400 l 2700 5550 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 2400 4050 m 3000 3450 l gs col0 s gr % Polyline n 600 4050 m 1200 3450 l 3000 3450 l 3000 5250 l 2400 5850 l gs col0 s gr % Polyline n 7950 1050 m 8550 450 l gs col0 s gr % Polyline n 3375 4050 m 3975 3450 l 5775 3450 l 5775 5250 l 5175 5850 l gs col0 s gr % Polyline n 600 7050 m 2400 7050 l 2400 8850 l 600 8850 l cp gs col0 s gr % Polyline 7.500 slw n 3975 8550 m 4200 8550 l 4200 8775 l 3975 8775 l cp gs 0.60 setgray ef gr gs col0 s gr % Polyline n 3975 8100 m 4200 8100 l 4200 8325 l 3975 8325 l cp gs 0.75 setgray ef gr gs col0 s gr % Polyline n 3975 7650 m 4200 7650 l 4200 7875 l 3975 7875 l cp gs 0.90 setgray ef gr gs col0 s gr % Polyline n 3525 6900 m 8400 6900 l 8400 9075 l 3525 9075 l cp gs col0 s gr % Polyline n 3975 7200 m 4200 7200 l 4200 7425 l 3975 7425 l cp gs 1.00 setgray ef gr gs col0 s gr % Polyline n 6300 8550 m 6525 8550 l 6525 8775 l 6300 8775 l cp gs 0.00 setgray ef gr gs col0 s gr % Polyline n 6300 8100 m 6525 8100 l 6525 8325 l 6300 8325 l cp gs 0.15 setgray ef gr gs col0 s gr % Polyline n 6300 7650 m 6525 7650 l 6525 7875 l 6300 7875 l cp gs 0.30 setgray ef gr gs col0 s gr % Polyline n 6300 7200 m 6525 7200 l 6525 7425 l 6300 7425 l cp gs 0.45 setgray ef gr gs col0 s gr % Polyline 15.000 slw n 600 1050 m 1200 450 l 3000 450 l 3000 2250 l 2400 2850 l gs col0 s gr /Times-Roman ff 270.00 scf sf 900 9150 m gs 1 -1 sc (ZYX Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 4350 7425 m gs 1 -1 sc (Processor 0) col0 sh gr /Times-Roman ff 270.00 scf sf 3900 3150 m gs 1 -1 sc (Y Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 1050 3150 m gs 1 -1 sc (Z Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 6600 3150 m gs 1 -1 sc (X Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 6525 6150 m gs 1 -1 sc (YX Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 975 6150 m gs 1 -1 sc (ZY Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 3825 6150 m gs 1 -1 sc (ZX Partition) col0 sh gr /Times-Roman ff 270.00 scf sf 6675 8775 m gs 1 -1 sc (Processor 7) col0 sh gr /Times-Roman ff 270.00 scf sf 4350 7875 m gs 1 -1 sc (Processor 1) col0 sh gr /Times-Roman ff 270.00 scf sf 4350 8325 m gs 1 -1 sc (Processor 2) col0 sh gr /Times-Roman ff 270.00 scf sf 4350 8775 m gs 1 -1 sc (Processor 3) col0 sh gr /Times-Roman ff 270.00 scf sf 6675 7425 m gs 1 -1 sc (Processor 4) col0 sh gr /Times-Roman ff 270.00 scf sf 6675 7875 m gs 1 -1 sc (Processor 5) col0 sh gr /Times-Roman ff 270.00 scf sf 6675 8325 m gs 1 -1 sc (Processor 6) col0 sh gr /Times-Bold ff 270.00 scf sf 975 1350 m gs 1 -1 sc (X) col0 sh gr % Polyline 30.000 slw gs clippath 944 945 m 1088 975 l 944 1005 l 1170 1005 l 1170 945 l cp 480 1469 m 450 1613 l 420 1469 l 420 1695 l 480 1695 l cp clip n 450 1650 m 450 975 l 1125 975 l gs col0 s gr gr % arrowhead 15.000 slw n 480 1469 m 450 1613 l 420 1469 l 450 1493 l 480 1469 l cp gs 0.00 setgray ef gr col0 s % arrowhead n 944 945 m 1088 975 l 944 1005 l 968 975 l 944 945 l cp gs 0.00 setgray ef gr col0 s $F2psEnd rs %%EndDocument @endspecial 2142 2325 a Fe(Figure)15 b(5.)2496 2226 y SDict begin H.S end 2496 2226 a 2496 2226 a SDict begin H.R end 2496 2226 a 2496 2226 a SDict begin [ /View [/XYZ H.V] /Dest (figure.5) cvn H.B /DEST pdfmark end 2496 2226 a 99 x Fe(V)-5 b(arious)16 b(3\255D)f(arra)n(y)i(par)r(titions)e (on)g(8)h(pr)n(o\255)2142 2425 y(cessor)o(s)2042 2702 y SDict begin H.S end 2042 2702 a 2042 2702 a SDict begin 12 H.A end 2042 2702 a 2042 2702 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.5.1) cvn H.B /DEST pdfmark end 2042 2702 a 83 x Ff(5.1)91 b(Scalability)23 b(Analysis)2142 3020 y Fg(W)-7 b(e)20 b(wrote)f(a)h(test)g(code)e(\(in)h(C)h (language\))d(to)j(e)n(v)n(aluate)e(the)h(per)n(-)2042 3120 y(formance)e(of)h(the)g(current)g(implementation)e(of)i(parallel)g (netCDF)-7 b(.)2042 3219 y(This)46 b(test)g(code)f(w)o(as)i(originally) d(de)n(v)o(eloped)f(in)j(F)o(ortran)e(by)2042 3319 y(W)-7 b(oo-sun)27 b(Y)-8 b(ang)27 b(and)g(Chris)h(Ding)f(at)h(La)o(wrence)f (Berk)o(ele)o(y)g(Na-)2042 3419 y(tional)g(Laboratory)d(\(LBL\).)i (Basically)h(it)h(reads/writes)e(a)h(three-)2042 3518 y(dimensional)17 b(array)g(\002eld)h(tt\(Z,Y)-11 b(,X\))17 b(from/into)g(a)h(single)g(netCDF)2042 3618 y(\002le,)34 b(where)c(Z=le)n(v)o(el)g(is)i(the)f(most)g(signi\002cant)f(dimension)f (and)2042 3717 y(X=longitude)k(is)j(the)f(least)g(signi\002cant)g (dimension.)67 b(The)35 b(test)2042 3817 y(code)27 b(partitions)f(the)h (three)f(dimensional)g(array)g(along)g(Z,)h(Y)-11 b(,)27 b(X,)2042 3917 y(ZY)-11 b(,)26 b(ZX,)g(YX,)g(and)f(ZYX)h(ax)o(es,)h (respecti)n(v)o(ely)-5 b(,)25 b(as)h(illustrated)f(in)2042 4016 y(Figure)2289 4017 y SDict begin H.S end 2289 4017 a -1 x Fg(5)2331 3960 y SDict begin H.R end 2331 3960 a 2331 4016 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.5) cvn H.B /ANN pdfmark end 2331 4016 a Fg(.)53 b(All)30 b(data)g(I/O)g(operations)e(in)h(these)h(tests)h(used)e(col-)2042 4116 y(lecti)n(v)o(e)21 b(I/O.)g(F)o(or)g(comparison)e(purpose,)h(we)i (prepared)d(the)i(same)2042 4216 y(test)g(using)e(the)g(original)g (serial)g(netCDF)h(API)g(and)f(ran)g(it)i(in)e(serial)2042 4315 y(mode,)30 b(in)f(which)f(a)i(single)e(processor)g(reads/writes)g (the)h(whole)2042 4415 y(array)-5 b(.)2142 4528 y(Figure)2384 4529 y SDict begin H.S end 2384 4529 a -1 x Fg(6)2425 4471 y SDict begin H.R end 2425 4471 a 2425 4528 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.6) cvn H.B /ANN pdfmark end 2425 4528 a 26 w Fg(sho)n(ws)24 b(the)h(performance)d (results)j(for)f(reading)f(and)2042 4628 y(writing)i(64)g(MB)h(and)f(1) g(GB)h(netCDF)g(datasets.)41 b(Generally)-5 b(,)25 b(the)2042 4727 y(parallel)15 b(netCDF)h(performance)d(scales)j(with)g(the)g (number)d(of)j(pro-)2042 4827 y(cesses.)40 b(Because)25 b(of)g(collecti)n(v)o(e)f(I/O)h(optimization,)f(the)h(perfor)n(-)2042 4926 y(mance)d(dif)n(ference)e(made)h(by)h(v)n(arious)f(access)i (patterns)e(is)i(small,)2042 5026 y(although)15 b(partitioning)f(in)j (the)g(Z)f(dimension)f(generally)g(performs)2042 5126 y(better)38 b(than)g(in)g(the)g(X)g(dimension)f(because)g(of)h(the)g (dif)n(ferent)2042 5225 y(access)31 b(contiguity)-5 b(.)52 b(The)29 b(o)o(v)o(erhead)e(in)m(v)n(olv)o(ed)h(is)j(inter)n(-process) 2042 5325 y(communication,)24 b(which)h(is)h(ne)o(gligible)e(comparing) f(to)i(the)h(disk)1928 5574 y(7)p eop %%Page: 8 8 8 7 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.8) cvn H.B /DEST pdfmark end -114 0 a 92 2101 a @beginspecial 0 @llx 0 @lly 979 @urx 554 @ury 4455 @rwi @setspecial %%BeginDocument: lblbenchnetcdf.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def /col32 {0.800 0.800 0.800 srgb} bind def /col33 {0.000 0.000 0.000 srgb} bind def end save -26.0 582.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 10694 m -1000 -1000 l 17743 -1000 l 17743 10694 l cp clip 0.06000 0.06000 sc /Times-Bold ff 270.00 scf sf 12900 8775 m gs 1 -1 sc (Parallel netCDF) col0 sh gr % Polyline 45.000 slw [60 30 15 30] 0 sd n 1425 3700 m 1575 3600 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 1425 3600 m 1575 3700 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 9825 3900 m 9975 3800 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 9825 8860 m 9975 8760 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 1425 8150 m 1575 8050 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 1425 8050 m 1575 8150 l gs col0 s gr [] 0 sd % Polyline 7.500 slw gs clippath 1681 3536 m 1575 3600 l 1639 3494 l 1543 3589 l 1586 3632 l cp clip n 2025 3150 m 1575 3600 l gs col0 s gr gr % arrowhead n 1681 3536 m 1575 3600 l 1639 3494 l 1660 3515 l 1681 3536 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 10091 3783 m 9975 3825 l 10058 3733 l 9946 3808 l 9979 3858 l cp clip n 10425 3525 m 9975 3825 l gs col0 s gr gr % arrowhead n 10091 3783 m 9975 3825 l 10058 3733 l 10075 3758 l 10091 3783 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 1694 8067 m 1575 8100 l 1664 8014 l 1547 8081 l 1577 8133 l cp clip n 2100 7800 m 1575 8100 l gs col0 s gr gr % arrowhead n 1694 8067 m 1575 8100 l 1664 8014 l 1679 8040 l 1694 8067 l cp gs 0.00 setgray ef gr col0 s /Times-Bold ff 270.00 scf sf 12900 3975 m gs 1 -1 sc (Parallel netCDF) col0 sh gr % Polyline gs clippath 10077 8704 m 9975 8775 l 10031 8665 l 9943 8767 l 9988 8806 l cp clip n 10500 8175 m 9975 8775 l gs col0 s gr gr % arrowhead n 10077 8704 m 9975 8775 l 10031 8665 l 10054 8685 l 10077 8704 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 2970 8730 m 2850 8700 l 2970 8670 l 2835 8670 l 2835 8730 l cp clip n 2850 8700 m 4425 8700 l gs col0 s gr gr % arrowhead n 2970 8730 m 2850 8700 l 2970 8670 l 2970 8700 l 2970 8730 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 8055 8670 m 8175 8700 l 8055 8730 l 8190 8730 l 8190 8670 l cp clip n 6450 8700 m 8175 8700 l gs col0 s gr gr % arrowhead n 8055 8670 m 8175 8700 l 8055 8730 l 8055 8700 l 8055 8670 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 2970 3930 m 2850 3900 l 2970 3870 l 2835 3870 l 2835 3930 l cp clip n 2850 3900 m 4425 3900 l gs col0 s gr gr % arrowhead n 2970 3930 m 2850 3900 l 2970 3870 l 2970 3900 l 2970 3930 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 8055 3870 m 8175 3900 l 8055 3930 l 8190 3930 l 8190 3870 l cp clip n 6450 3900 m 8175 3900 l gs col0 s gr gr % arrowhead n 8055 3870 m 8175 3900 l 8055 3930 l 8055 3900 l 8055 3870 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 11370 3930 m 11250 3900 l 11370 3870 l 11235 3870 l 11235 3930 l cp clip n 11250 3900 m 12825 3900 l gs col0 s gr gr % arrowhead n 11370 3930 m 11250 3900 l 11370 3870 l 11370 3900 l 11370 3930 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 16455 3870 m 16575 3900 l 16455 3930 l 16590 3930 l 16590 3870 l cp clip n 14850 3900 m 16575 3900 l gs col0 s gr gr % arrowhead n 16455 3870 m 16575 3900 l 16455 3930 l 16455 3900 l 16455 3870 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 11370 8730 m 11250 8700 l 11370 8670 l 11235 8670 l 11235 8730 l cp clip n 11250 8700 m 12825 8700 l gs col0 s gr gr % arrowhead n 11370 8730 m 11250 8700 l 11370 8670 l 11370 8700 l 11370 8730 l cp gs 0.00 setgray ef gr col0 s % Polyline gs clippath 16455 8670 m 16575 8700 l 16455 8730 l 16590 8730 l 16590 8670 l cp clip n 14850 8700 m 16575 8700 l gs col0 s gr gr % arrowhead n 16455 8670 m 16575 8700 l 16455 8730 l 16455 8700 l 16455 8670 l cp gs 0.00 setgray ef gr col0 s % Polyline 45.000 slw [60 30 15 30] 0 sd n 9825 3800 m 9975 3900 l gs col0 s gr [] 0 sd % Polyline [60 30 15 30] 0 sd n 9825 8760 m 9975 8860 l gs col0 s gr [] 0 sd /Times-Bold ff 270.00 scf sf 4500 3975 m gs 1 -1 sc (Parallel netCDF) col0 sh gr /Times-Bold ff 270.00 scf sf 4500 8775 m gs 1 -1 sc (Parallel netCDF) col0 sh gr % Polyline 7.500 slw n 1500 3719 m 8190 3719 l gs col32 s gr % Polyline n 4172 4177 m 4172 973 l gs col32 s gr % Polyline n 1500 1431 m 8190 1431 l gs col32 s gr % Polyline n 5511 4177 m 5511 973 l gs col32 s gr % Polyline n 6851 4177 m 6851 973 l gs col32 s gr % Polyline n 1500 2346 m 8190 2346 l gs col32 s gr % Polyline n 1500 1888 m 8190 1888 l gs col32 s gr % Polyline n 2850 4177 m 2850 973 l gs col32 s gr % Polyline n 13911 4177 m 13911 973 l gs col32 s gr % Polyline n 15251 4177 m 15251 973 l gs col32 s gr % Polyline n 9900 3376 m 16590 3376 l gs col32 s gr % Polyline n 9900 2575 m 16590 2575 l gs col32 s gr % Polyline n 11250 4177 m 11250 973 l gs col32 s gr % Polyline n 3904 8977 m 3904 5773 l gs col32 s gr % Polyline n 4975 8977 m 4975 5773 l gs col32 s gr % Polyline n 1500 7375 m 8190 7375 l gs col32 s gr % Polyline n 1500 7909 m 8190 7909 l gs col32 s gr % Polyline n 6047 8977 m 6047 5773 l gs col32 s gr % Polyline n 7118 8977 m 7118 5773 l gs col32 s gr % Polyline n 1500 6841 m 8190 6841 l gs col32 s gr % Polyline n 15518 8977 m 15518 5773 l gs col32 s gr % Polyline n 14447 8977 m 14447 5773 l gs col32 s gr % Polyline n 13375 8977 m 13375 5773 l gs col32 s gr % Polyline n 12304 8977 m 12304 5773 l gs col32 s gr % Polyline n 9900 7055 m 16590 7055 l gs col32 s gr % Polyline n 9900 6414 m 16590 6414 l gs col32 s gr % Polyline n 9900 7695 m 16590 7695 l gs col32 s gr % Polyline n 9900 8336 m 16590 8336 l gs col32 s gr % Polyline n 11250 8977 m 11250 5773 l gs col32 s gr % Polyline n 1500 2804 m 8190 2804 l gs col32 s gr % Polyline n 1500 6307 m 8190 6307 l gs col32 s gr % Polyline n 9900 8977 m 9900 5773 l gs col32 s gr % Polyline n 12572 4177 m 12572 973 l gs col32 s gr % Polyline n 1500 8443 m 8190 8443 l gs col32 s gr % Polyline n 2850 8977 m 2850 5773 l gs col32 s gr % Polyline n 1500 3262 m 8190 3262 l gs col32 s gr % Polyline n 9900 1774 m 16590 1774 l gs col32 s gr % Polyline n 8153 2745 m 8227 2819 l gs col5 s gr % Polyline n 2795 3742 m 2869 3668 l gs col5 s gr % Polyline n 2795 3668 m 2869 3742 l gs col5 s gr % Polyline n 4135 3625 m 4209 3625 l gs col5 s gr % Polyline n 4172 3662 m 4172 3588 l gs col5 s gr % Polyline n 4135 3662 m 4209 3588 l gs col5 s gr % Polyline n 4135 3588 m 4209 3662 l gs col5 s gr % Polyline n 5474 3167 m 5548 3167 l gs col5 s gr % Polyline n 5511 3204 m 5511 3130 l gs col5 s gr % Polyline n 5474 3130 m 5548 3204 l gs col5 s gr % Polyline n 6814 2826 m 6888 2826 l gs col5 s gr % Polyline n 6851 2863 m 6851 2789 l gs col5 s gr % Polyline n 6814 2863 m 6888 2789 l gs col5 s gr % Polyline n 6814 2789 m 6888 2863 l gs col5 s gr % Polyline n 8153 2782 m 8227 2782 l gs col5 s gr % Polyline n 8190 2819 m 8190 2745 l gs col5 s gr % Polyline n 5474 3204 m 5548 3130 l gs col5 s gr % Polyline n 8153 2819 m 8227 2745 l gs col5 s gr % Polyline n 2832 3705 m 4172 3426 l 5511 3211 l 6851 2868 l 8190 2854 l gs col6 s gr % Polyline n 2795 3705 m 2832 3742 l 2869 3705 l 2832 3668 l 2795 3705 l cp gs col6 s gr % Polyline n 2828 3705 m 2836 3705 l gs col6 s gr % Polyline n 4135 3426 m 4172 3463 l 4209 3426 l 4172 3389 l 4135 3426 l cp gs col6 s gr % Polyline n 4168 3426 m 4176 3426 l gs col6 s gr % Polyline n 5474 3211 m 5511 3248 l 5548 3211 l 5511 3174 l 5474 3211 l cp gs col6 s gr % Polyline n 5507 3211 m 5515 3211 l gs col6 s gr % Polyline n 6814 2868 m 6851 2905 l 6888 2868 l 6851 2831 l 6814 2868 l cp gs col6 s gr % Polyline n 6847 2868 m 6855 2868 l gs col6 s gr % Polyline n 8153 2854 m 8190 2891 l 8227 2854 l 8190 2817 l 8153 2854 l cp gs col6 s gr % Polyline n 8186 2854 m 8194 2854 l gs col6 s gr % Polyline n 8190 2073 m 8190 1999 l gs col1 s gr % Polyline n 8153 2036 m 8227 2036 l gs col1 s gr % Polyline n 6851 1902 m 6851 1828 l gs col1 s gr % Polyline n 6814 1865 m 6888 1865 l gs col1 s gr % Polyline n 5511 2983 m 5511 2909 l gs col1 s gr % Polyline n 2832 3742 m 2832 3668 l gs col5 s gr % Polyline n 5474 2946 m 5548 2946 l gs col1 s gr % Polyline n 4135 3590 m 4209 3590 l gs col1 s gr % Polyline n 2832 3742 m 2832 3668 l gs col1 s gr % Polyline n 2795 3705 m 2869 3705 l gs col1 s gr % Polyline n 2832 3705 m 4172 3590 l 5511 2946 l 6851 1865 l 8190 2036 l gs col1 s gr % Polyline n 8186 1389 m 8194 1389 l gs col0 s gr % Polyline n 8153 1389 m 8190 1426 l 8227 1389 l 8190 1352 l 8153 1389 l cp gs col0 s gr % Polyline n 1482 3719 m 1557 3719 l gs col0 s gr % Polyline n 1482 3262 m 1557 3262 l gs col0 s gr % Polyline n 1482 2804 m 1557 2804 l gs col0 s gr % Polyline n 1482 2346 m 1557 2346 l gs col0 s gr % Polyline n 1482 1888 m 1557 1888 l gs col0 s gr % Polyline n 1482 1431 m 1557 1431 l gs col0 s gr % Polyline n 1482 973 m 1557 973 l gs col0 s gr % Polyline [15 60] 60 sd n 1482 4177 m 1482 973 l gs col0 s gr [] 0 sd % Polyline n 1482 4177 m 1482 4102 l gs col0 s gr % Polyline n 1482 973 m 1482 1048 l gs col0 s gr % Polyline n 1478 3705 m 1486 3705 l gs col0 s gr % Polyline n 1482 4177 m 1557 4177 l gs col0 s gr % Polyline n 11232 2753 m 12572 2594 l 13911 2359 l 15251 2787 l 16590 2132 l gs col0 s gr % Polyline n 11195 2753 m 11232 2790 l 11269 2753 l 11232 2716 l 11195 2753 l cp gs col0 s gr % Polyline n 11228 2753 m 11236 2753 l gs col0 s gr % Polyline n 12535 2594 m 12572 2631 l 12609 2594 l 12572 2557 l 12535 2594 l cp gs col0 s gr % Polyline n 12568 2594 m 12576 2594 l gs col0 s gr % Polyline n 13874 2359 m 13911 2396 l 13948 2359 l 13911 2322 l 13874 2359 l cp gs col0 s gr % Polyline n 13907 2359 m 13915 2359 l gs col0 s gr % Polyline n 15214 2787 m 15251 2824 l 15288 2787 l 15251 2750 l 15214 2787 l cp gs col0 s gr % Polyline n 15247 2787 m 15255 2787 l gs col0 s gr % Polyline n 6847 2055 m 6855 2055 l gs col0 s gr % Polyline n 6814 2055 m 6851 2092 l 6888 2055 l 6851 2018 l 6814 2055 l cp gs col0 s gr % Polyline n 11232 2753 m 12572 3011 l 13911 3022 l 15251 3022 l 16590 2132 l gs col1 s gr % Polyline n 11195 2753 m 11269 2753 l gs col1 s gr % Polyline n 11232 2790 m 11232 2716 l gs col1 s gr % Polyline n 12535 3011 m 12609 3011 l gs col1 s gr % Polyline n 12572 3048 m 12572 2974 l gs col1 s gr % Polyline n 2795 3705 m 2869 3705 l gs col5 s gr % Polyline n 2832 3705 m 4172 3625 l 5511 3167 l 6851 2826 l 8190 2782 l gs col5 s gr % Polyline n 8186 2851 m 8194 2851 l gs col4 s gr % Polyline n 4172 3627 m 4172 3553 l gs col1 s gr % Polyline n 8190 2802 m 8141 2875 l 8239 2875 l 8190 2802 l cp gs col4 s gr % Polyline n 13874 3022 m 13948 3022 l gs col1 s gr % Polyline n 11232 2753 m 12572 3120 l 13911 2499 l 15251 2460 l 16590 2800 l gs col2 s gr % Polyline n 11195 2790 m 11269 2790 l 11269 2716 l 11195 2716 l 11195 2790 l cp gs col2 s gr % Polyline n 11228 2753 m 11236 2753 l gs col2 s gr % Polyline n 12535 3157 m 12609 3157 l 12609 3083 l 12535 3083 l 12535 3157 l cp gs col2 s gr % Polyline n 12568 3120 m 12576 3120 l gs col2 s gr % Polyline n 13874 2536 m 13948 2536 l 13948 2462 l 13874 2462 l 13874 2536 l cp gs col2 s gr % Polyline n 13907 2499 m 13915 2499 l gs col2 s gr % Polyline n 15214 2497 m 15288 2497 l 15288 2423 l 15214 2423 l 15214 2497 l cp gs col2 s gr % Polyline n 15247 2460 m 15255 2460 l gs col2 s gr % Polyline n 16553 2837 m 16627 2837 l 16627 2763 l 16553 2763 l 16553 2837 l cp gs col2 s gr % Polyline n 16586 2800 m 16594 2800 l gs col2 s gr % Polyline n 11232 2753 m 12572 2560 l 13911 2193 l 15251 1548 l 16590 2227 l gs col3 s gr % Polyline n 11195 2790 m 11269 2716 l gs col3 s gr % Polyline n 11195 2716 m 11269 2790 l gs col3 s gr % Polyline n 12535 2597 m 12609 2523 l gs col3 s gr % Polyline n 12535 2523 m 12609 2597 l gs col3 s gr % Polyline n 13874 2230 m 13948 2156 l gs col3 s gr % Polyline n 13874 2156 m 13948 2230 l gs col3 s gr % Polyline n 15214 1585 m 15288 1511 l gs col3 s gr % Polyline n 15214 1511 m 15288 1585 l gs col3 s gr % Polyline n 16553 2264 m 16627 2190 l gs col3 s gr % Polyline n 16553 2190 m 16627 2264 l gs col3 s gr % Polyline n 11232 2753 m 12572 2738 l 13911 2501 l 15251 1889 l 16590 2882 l gs col4 s gr % Polyline n 11232 2704 m 11183 2777 l 11281 2777 l 11232 2704 l cp gs col4 s gr % Polyline n 11228 2753 m 11236 2753 l gs col4 s gr % Polyline n 12572 2689 m 12523 2762 l 12621 2762 l 12572 2689 l cp gs col4 s gr % Polyline n 12568 2738 m 12576 2738 l gs col4 s gr % Polyline n 13911 2452 m 13862 2525 l 13960 2525 l 13911 2452 l cp gs col4 s gr % Polyline n 13907 2501 m 13915 2501 l gs col4 s gr % Polyline n 15251 1840 m 15202 1913 l 15300 1913 l 15251 1840 l cp gs col4 s gr % Polyline n 15247 1889 m 15255 1889 l gs col4 s gr % Polyline n 16590 2833 m 16541 2906 l 16639 2906 l 16590 2833 l cp gs col4 s gr % Polyline n 16586 2882 m 16594 2882 l gs col4 s gr % Polyline n 11232 2753 m 12572 3296 l 13911 2664 l 15251 2623 l 16590 3238 l gs col5 s gr % Polyline n 11195 2753 m 11269 2753 l gs col5 s gr % Polyline n 11232 2790 m 11232 2716 l gs col5 s gr % Polyline n 11195 2790 m 11269 2716 l gs col5 s gr % Polyline n 11195 2716 m 11269 2790 l gs col5 s gr % Polyline n 12535 3296 m 12609 3296 l gs col5 s gr % Polyline n 12572 3333 m 12572 3259 l gs col5 s gr % Polyline n 12535 3333 m 12609 3259 l gs col5 s gr % Polyline n 12535 3259 m 12609 3333 l gs col5 s gr % Polyline n 13874 2664 m 13948 2664 l gs col5 s gr % Polyline n 13911 2701 m 13911 2627 l gs col5 s gr % Polyline n 13874 2701 m 13948 2627 l gs col5 s gr % Polyline n 13874 2627 m 13948 2701 l gs col5 s gr % Polyline n 15214 2623 m 15288 2623 l gs col5 s gr % Polyline n 15251 2660 m 15251 2586 l gs col5 s gr % Polyline n 15214 2660 m 15288 2586 l gs col5 s gr % Polyline n 15214 2586 m 15288 2660 l gs col5 s gr % Polyline n 16553 3238 m 16627 3238 l gs col5 s gr % Polyline n 16590 3275 m 16590 3201 l gs col5 s gr % Polyline n 16553 3275 m 16627 3201 l gs col5 s gr % Polyline n 16553 3201 m 16627 3275 l gs col5 s gr % Polyline n 11232 2753 m 12572 2638 l 13911 2485 l 15251 2403 l 16590 2534 l gs col6 s gr % Polyline n 11195 2753 m 11232 2790 l 11269 2753 l 11232 2716 l 11195 2753 l cp gs col6 s gr % Polyline n 11228 2753 m 11236 2753 l gs col6 s gr % Polyline n 12535 2638 m 12572 2675 l 12609 2638 l 12572 2601 l 12535 2638 l cp gs col6 s gr % Polyline n 12568 2638 m 12576 2638 l gs col6 s gr % Polyline n 13874 2485 m 13911 2522 l 13948 2485 l 13911 2448 l 13874 2485 l cp gs col6 s gr % Polyline n 13907 2485 m 13915 2485 l gs col6 s gr % Polyline n 15214 2403 m 15251 2440 l 15288 2403 l 15251 2366 l 15214 2403 l cp gs col6 s gr % Polyline n 15247 2403 m 15255 2403 l gs col6 s gr % Polyline n 16553 2534 m 16590 2571 l 16627 2534 l 16590 2497 l 16553 2534 l cp gs col6 s gr % Polyline n 16586 2534 m 16594 2534 l gs col6 s gr % Polyline n 5507 3220 m 5515 3220 l gs col0 s gr % Polyline n 5474 3220 m 5511 3257 l 5548 3220 l 5511 3183 l 5474 3220 l cp gs col0 s gr % Polyline n 4168 3433 m 4176 3433 l gs col0 s gr % Polyline n 4135 3433 m 4172 3470 l 4209 3433 l 4172 3396 l 4135 3433 l cp gs col0 s gr % Polyline n 2828 3705 m 2836 3705 l gs col0 s gr % Polyline n 6847 2744 m 6855 2744 l gs col4 s gr % Polyline n 6851 2695 m 6802 2768 l 6900 2768 l 6851 2695 l cp gs col4 s gr % Polyline n 5507 3348 m 5515 3348 l gs col4 s gr % Polyline n 5511 3299 m 5462 3372 l 5560 3372 l 5511 3299 l cp gs col4 s gr % Polyline n 13911 3059 m 13911 2985 l gs col1 s gr % Polyline n 2795 3705 m 2832 3742 l 2869 3705 l 2832 3668 l 2795 3705 l cp gs col0 s gr % Polyline n 4168 3425 m 4176 3425 l gs col4 s gr % Polyline n 4172 3376 m 4123 3449 l 4221 3449 l 4172 3376 l cp gs col4 s gr % Polyline n 2828 3705 m 2836 3705 l gs col4 s gr % Polyline n 2832 3656 m 2783 3729 l 2881 3729 l 2832 3656 l cp gs col4 s gr % Polyline n 2832 3705 m 4172 3425 l 5511 3348 l 6851 2744 l 8190 2851 l gs col4 s gr % Polyline n 8153 2827 m 8227 2901 l gs col3 s gr % Polyline n 8153 2901 m 8227 2827 l gs col3 s gr % Polyline n 6814 2539 m 6888 2613 l gs col3 s gr % Polyline n 6814 2613 m 6888 2539 l gs col3 s gr % Polyline n 5474 3120 m 5548 3194 l gs col3 s gr % Polyline n 5474 3194 m 5548 3120 l gs col3 s gr % Polyline n 4135 3390 m 4209 3464 l gs col3 s gr % Polyline n 4135 3464 m 4209 3390 l gs col3 s gr % Polyline n 2795 3668 m 2869 3742 l gs col3 s gr % Polyline n 2795 3742 m 2869 3668 l gs col3 s gr % Polyline n 15214 3022 m 15288 3022 l gs col1 s gr % Polyline n 2832 3705 m 4172 3433 l 5511 3220 l 6851 2055 l 8190 1389 l gs col0 s gr % Polyline n 2832 3705 m 4172 3427 l 5511 3157 l 6851 2576 l 8190 2864 l gs col3 s gr % Polyline n 8186 2866 m 8194 2866 l gs col2 s gr % Polyline n 8153 2903 m 8227 2903 l 8227 2829 l 8153 2829 l 8153 2903 l cp gs col2 s gr % Polyline n 6847 2849 m 6855 2849 l gs col2 s gr % Polyline n 15251 3059 m 15251 2985 l gs col1 s gr % Polyline n 6814 2886 m 6888 2886 l 6888 2812 l 6814 2812 l 6814 2886 l cp gs col2 s gr % Polyline n 16553 2132 m 16627 2132 l gs col1 s gr % Polyline n 8190 1431 m 8115 1431 l gs col0 s gr % Polyline n 16553 2132 m 16590 2169 l 16627 2132 l 16590 2095 l 16553 2132 l cp gs col0 s gr % Polyline n 5507 3299 m 5515 3299 l gs col2 s gr % Polyline n 5474 3336 m 5548 3336 l 5548 3262 l 5474 3262 l 5474 3336 l cp gs col2 s gr % Polyline n 4168 3610 m 4176 3610 l gs col2 s gr % Polyline n 4135 3647 m 4209 3647 l 4209 3573 l 4135 3573 l 4135 3647 l cp gs col2 s gr % Polyline n 2828 3705 m 2836 3705 l gs col2 s gr % Polyline n 2795 3742 m 2869 3742 l 2869 3668 l 2795 3668 l 2795 3742 l cp gs col2 s gr % Polyline n 16590 2169 m 16590 2095 l gs col1 s gr % Polyline n 2832 3705 m 4172 3610 l 5511 3299 l 6851 2849 l 8190 2866 l gs col2 s gr % Polyline n 16586 2132 m 16594 2132 l gs col0 s gr % Polyline n 8190 2804 m 8115 2804 l gs col0 s gr % Polyline n 2832 3719 m 2907 3719 l gs col0 s gr % Polyline n 6047 7236 m 5998 7309 l 6096 7309 l 6047 7236 l cp gs col4 s gr % Polyline n 7081 6435 m 7155 6435 l gs col1 s gr % Polyline n 7118 6472 m 7118 6398 l gs col1 s gr % Polyline n 1482 5773 m 1557 5773 l gs col0 s gr % Polyline n 1482 6307 m 1557 6307 l gs col0 s gr % Polyline n 1482 6841 m 1557 6841 l gs col0 s gr % Polyline n 1482 7375 m 1557 7375 l gs col0 s gr % Polyline n 1482 7909 m 1557 7909 l gs col0 s gr % Polyline n 1482 8443 m 1557 8443 l gs col0 s gr % Polyline n 1482 8977 m 1557 8977 l gs col0 s gr % Polyline n 1482 5773 m 1482 5848 l gs col0 s gr % Polyline n 2828 8145 m 2836 8145 l gs col6 s gr % Polyline n 2832 8182 m 2832 8108 l gs col5 s gr % Polyline n 2828 8145 m 2836 8145 l gs col4 s gr % Polyline n 2828 8145 m 2836 8145 l gs col2 s gr % Polyline n 2832 8182 m 2832 8108 l gs col1 s gr % Polyline n 2828 8145 m 2836 8145 l gs col0 s gr % Polyline n 2795 8145 m 2832 8182 l 2869 8145 l 2832 8108 l 2795 8145 l cp gs col6 s gr % Polyline n 2795 8108 m 2869 8182 l gs col5 s gr % Polyline n 2795 8182 m 2869 8108 l gs col5 s gr % Polyline n 2795 8145 m 2869 8145 l gs col5 s gr % Polyline n 2832 8096 m 2783 8169 l 2881 8169 l 2832 8096 l cp gs col4 s gr % Polyline n 2795 8108 m 2869 8182 l gs col3 s gr % Polyline n 2795 8182 m 2869 8108 l gs col3 s gr % Polyline n 2795 8182 m 2869 8182 l 2869 8108 l 2795 8108 l 2795 8182 l cp gs col2 s gr % Polyline n 2795 8145 m 2869 8145 l gs col1 s gr % Polyline n 2795 8145 m 2832 8182 l 2869 8145 l 2832 8108 l 2795 8145 l cp gs col0 s gr % Polyline n 9882 8977 m 9957 8977 l gs col0 s gr % Polyline n 9882 8336 m 9957 8336 l gs col0 s gr % Polyline n 9882 7695 m 9957 7695 l gs col0 s gr % Polyline n 9882 7055 m 9957 7055 l gs col0 s gr % Polyline n 9900 4177 m 16590 4177 l 16590 973 l 9900 975 l 9900 4177 l cp gs col0 s gr % Polyline n 9882 6414 m 9957 6414 l gs col0 s gr % Polyline n 9882 5773 m 9957 5773 l gs col0 s gr % Polyline n 9882 8977 m 9882 8902 l gs col0 s gr % Polyline n 16590 2575 m 16515 2575 l gs col0 s gr % Polyline n 9882 5773 m 9882 5848 l gs col0 s gr % Polyline [15 60] 60 sd n 9882 8977 m 9882 5773 l gs col0 s gr [] 0 sd % Polyline n 4172 4177 m 4172 4102 l gs col0 s gr % Polyline n 5511 4177 m 5511 4102 l gs col0 s gr % Polyline n 6851 4177 m 6851 4102 l gs col0 s gr % Polyline n 16586 6834 m 16594 6834 l gs col6 s gr % Polyline n 16553 6834 m 16590 6871 l 16627 6834 l 16590 6797 l 16553 6834 l cp gs col6 s gr % Polyline n 15514 7034 m 15522 7034 l gs col6 s gr % Polyline n 15481 7034 m 15518 7071 l 15555 7034 l 15518 6997 l 15481 7034 l cp gs col6 s gr % Polyline n 14443 6860 m 14451 6860 l gs col6 s gr % Polyline n 14410 6860 m 14447 6897 l 14484 6860 l 14447 6823 l 14410 6860 l cp gs col6 s gr % Polyline n 13371 7134 m 13379 7134 l gs col6 s gr % Polyline n 13338 7134 m 13375 7171 l 13412 7134 l 13375 7097 l 13338 7134 l cp gs col6 s gr % Polyline n 12300 7180 m 12308 7180 l gs col6 s gr % Polyline n 12267 7180 m 12304 7217 l 12341 7180 l 12304 7143 l 12267 7180 l cp gs col6 s gr % Polyline n 11232 7523 m 12304 7180 l 13375 7134 l 14447 6860 l 15518 7034 l 16590 6834 l gs col6 s gr % Polyline n 16553 6634 m 16627 6708 l gs col5 s gr % Polyline n 16553 6708 m 16627 6634 l gs col5 s gr % Polyline n 16590 6708 m 16590 6634 l gs col5 s gr % Polyline n 16553 6671 m 16627 6671 l gs col5 s gr % Polyline n 15481 6901 m 15555 6975 l gs col5 s gr % Polyline n 15481 6975 m 15555 6901 l gs col5 s gr % Polyline n 15518 6975 m 15518 6901 l gs col5 s gr % Polyline n 15481 6938 m 15555 6938 l gs col5 s gr % Polyline n 14410 6932 m 14484 7006 l gs col5 s gr % Polyline n 14410 7006 m 14484 6932 l gs col5 s gr % Polyline n 14447 7006 m 14447 6932 l gs col5 s gr % Polyline n 14410 6969 m 14484 6969 l gs col5 s gr % Polyline n 13338 6926 m 13412 7000 l gs col5 s gr % Polyline n 13338 7000 m 13412 6926 l gs col5 s gr % Polyline n 16586 6366 m 16594 6366 l gs col0 s gr % Polyline n 13375 7000 m 13375 6926 l gs col5 s gr % Polyline n 13338 6963 m 13412 6963 l gs col5 s gr % Polyline n 12267 7024 m 12341 7098 l gs col5 s gr % Polyline n 12267 7098 m 12341 7024 l gs col5 s gr % Polyline n 12304 7098 m 12304 7024 l gs col5 s gr % Polyline n 12267 7061 m 12341 7061 l gs col5 s gr % Polyline n 11196 2753 m 11233 2790 l 11270 2753 l 11233 2716 l 11196 2753 l cp gs col0 s gr % Polyline n 9878 2753 m 9886 2753 l gs col0 s gr % Polyline n 9882 973 m 9882 1048 l gs col0 s gr % Polyline n 9882 4177 m 9882 4102 l gs col0 s gr % Polyline [15 60] 60 sd n 9882 4177 m 9882 973 l gs col0 s gr [] 0 sd % Polyline n 9882 973 m 9957 973 l gs col0 s gr % Polyline n 9882 1774 m 9957 1774 l gs col0 s gr % Polyline n 11195 7523 m 11232 7560 l 11269 7523 l 11232 7486 l 11195 7523 l cp gs col0 s gr % Polyline n 11228 7523 m 11236 7523 l gs col0 s gr % Polyline n 11195 7523 m 11269 7523 l gs col1 s gr % Polyline n 11232 7560 m 11232 7486 l gs col1 s gr % Polyline n 11195 7560 m 11269 7560 l 11269 7486 l 11195 7486 l 11195 7560 l cp gs col2 s gr % Polyline n 11228 7523 m 11236 7523 l gs col2 s gr % Polyline n 11195 7560 m 11269 7486 l gs col3 s gr % Polyline n 11195 7486 m 11269 7560 l gs col3 s gr % Polyline n 11232 7474 m 11183 7547 l 11281 7547 l 11232 7474 l cp gs col4 s gr % Polyline n 11228 7523 m 11236 7523 l gs col4 s gr % Polyline n 11195 7523 m 11269 7523 l gs col5 s gr % Polyline n 11232 7560 m 11232 7486 l gs col5 s gr % Polyline n 11195 7560 m 11269 7486 l gs col5 s gr % Polyline n 11195 7486 m 11269 7560 l gs col5 s gr % Polyline n 11195 7523 m 11232 7560 l 11269 7523 l 11232 7486 l 11195 7523 l cp gs col6 s gr % Polyline n 11228 7523 m 11236 7523 l gs col6 s gr % Polyline n 11232 7523 m 12304 7061 l 13375 6963 l 14447 6969 l 15518 6938 l 16590 6671 l gs col5 s gr % Polyline n 16586 6707 m 16594 6707 l gs col4 s gr % Polyline n 16590 6658 m 16541 6731 l 16639 6731 l 16590 6658 l cp gs col4 s gr % Polyline n 15514 6518 m 15522 6518 l gs col4 s gr % Polyline n 15518 6469 m 15469 6542 l 15567 6542 l 15518 6469 l cp gs col4 s gr % Polyline n 14443 7073 m 14451 7073 l gs col4 s gr % Polyline n 14447 7024 m 14398 7097 l 14496 7097 l 14447 7024 l cp gs col4 s gr % Polyline n 13371 7231 m 13379 7231 l gs col4 s gr % Polyline n 13375 7182 m 13326 7255 l 13424 7255 l 13375 7182 l cp gs col4 s gr % Polyline n 12300 7222 m 12308 7222 l gs col4 s gr % Polyline n 12304 7173 m 12255 7246 l 12353 7246 l 12304 7173 l cp gs col4 s gr % Polyline n 11232 7523 m 12304 7222 l 13375 7231 l 14447 7073 l 15518 6518 l 16590 6707 l gs col4 s gr % Polyline n 16553 6909 m 16627 6983 l gs col3 s gr % Polyline n 16553 6983 m 16627 6909 l gs col3 s gr % Polyline n 15481 6431 m 15555 6505 l gs col3 s gr % Polyline n 15481 6505 m 15555 6431 l gs col3 s gr % Polyline n 14410 6843 m 14484 6917 l gs col3 s gr % Polyline n 14410 6917 m 14484 6843 l gs col3 s gr % Polyline n 13338 6991 m 13412 7065 l gs col3 s gr % Polyline n 13338 7065 m 13412 6991 l gs col3 s gr % Polyline n 12267 7263 m 12341 7337 l gs col3 s gr % Polyline n 12267 7337 m 12341 7263 l gs col3 s gr % Polyline n 11232 7523 m 12304 7300 l 13375 7028 l 14447 6880 l 15518 6468 l 16590 6946 l gs col3 s gr % Polyline n 16586 7401 m 16594 7401 l gs col2 s gr % Polyline n 16553 7438 m 16627 7438 l 16627 7364 l 16553 7364 l 16553 7438 l cp gs col2 s gr % Polyline n 15514 7287 m 15522 7287 l gs col2 s gr % Polyline n 15481 7324 m 15555 7324 l 15555 7250 l 15481 7250 l 15481 7324 l cp gs col2 s gr % Polyline n 14443 6869 m 14451 6869 l gs col2 s gr % Polyline n 14410 6906 m 14484 6906 l 14484 6832 l 14410 6832 l 14410 6906 l cp gs col2 s gr % Polyline n 13371 7225 m 13379 7225 l gs col2 s gr % Polyline n 8190 5773 m 8115 5773 l gs col0 s gr % Polyline n 9882 2575 m 9957 2575 l gs col0 s gr % Polyline n 9882 3376 m 9957 3376 l gs col0 s gr % Polyline n 13338 7262 m 13412 7262 l 13412 7188 l 13338 7188 l 13338 7262 l cp gs col2 s gr % Polyline n 9882 4177 m 9957 4177 l gs col0 s gr % Polyline n 12300 7506 m 12308 7506 l gs col2 s gr % Polyline [15 60] 60 sd n 1481 8979 m 1481 5775 l gs col0 s gr [] 0 sd % Polyline n 8190 3719 m 8115 3719 l gs col0 s gr % Polyline n 8190 3262 m 8115 3262 l gs col0 s gr % Polyline n 8190 2804 m 8115 2804 l gs col0 s gr % Polyline n 8190 2346 m 8115 2346 l gs col0 s gr % Polyline [15 60] 60 sd n 8116 1888 m 8190 1888 l gs col0 s gr [] 0 sd % Polyline n 8190 1888 m 8115 1888 l gs col0 s gr % Polyline [15 60] 60 sd n 8116 1431 m 8190 1431 l gs col0 s gr [] 0 sd % Polyline n 8190 1431 m 8115 1431 l gs col0 s gr % Polyline [15 60] 60 sd n 2832 973 m 8190 973 l gs col0 s gr [] 0 sd % Polyline n 8190 973 m 8115 973 l gs col0 s gr % Polyline n 4172 973 m 4172 1048 l gs col0 s gr % Polyline n 5511 973 m 5511 1048 l gs col0 s gr % Polyline n 6851 973 m 6851 1048 l gs col0 s gr % Polyline [15 60] 60 sd n 8190 4177 m 8190 973 l gs col0 s gr [] 0 sd % Polyline n 8190 973 m 8190 1048 l gs col0 s gr % Polyline n 12267 7543 m 12341 7543 l 12341 7469 l 12267 7469 l 12267 7543 l cp gs col2 s gr % Polyline n 11232 7523 m 12304 7506 l 13375 7225 l 14447 6869 l 15518 7287 l 16590 7401 l gs col2 s gr % Polyline n 16590 6949 m 16590 6875 l gs col1 s gr % Polyline n 16553 6912 m 16627 6912 l gs col1 s gr % Polyline n 15518 6737 m 15518 6663 l gs col1 s gr % Polyline n 15481 6700 m 15555 6700 l gs col1 s gr % Polyline n 14447 6820 m 14447 6746 l gs col1 s gr % Polyline n 14410 6783 m 14484 6783 l gs col1 s gr % Polyline n 13375 6512 m 13375 6438 l gs col1 s gr % Polyline n 13338 6475 m 13412 6475 l gs col1 s gr % Polyline n 12304 6895 m 12304 6821 l gs col1 s gr % Polyline [15 60] 60 sd n 11232 4177 m 16590 4177 l gs col0 s gr [] 0 sd % Polyline n 16590 4177 m 16515 4177 l gs col0 s gr % Polyline n 16590 3376 m 16515 3376 l gs col0 s gr % Polyline n 13371 7446 m 13379 7446 l gs col0 s gr % Polyline [15 60] 60 sd n 16516 1774 m 16590 1774 l gs col0 s gr [] 0 sd % Polyline n 16590 1774 m 16515 1774 l gs col0 s gr % Polyline [15 60] 60 sd n 11232 973 m 16590 973 l gs col0 s gr [] 0 sd % Polyline n 16590 973 m 16515 973 l gs col0 s gr % Polyline n 12572 4177 m 12572 4102 l gs col0 s gr % Polyline n 12572 973 m 12572 1048 l gs col0 s gr % Polyline n 13911 4177 m 13911 4102 l gs col0 s gr % Polyline n 13911 973 m 13911 1048 l gs col0 s gr % Polyline n 15251 4177 m 15251 4102 l gs col0 s gr % Polyline n 15251 973 m 15251 1048 l gs col0 s gr % Polyline [15 60] 60 sd n 16590 4177 m 16590 973 l gs col0 s gr [] 0 sd % Polyline n 16590 4177 m 16590 4102 l gs col0 s gr % Polyline n 16590 973 m 16590 1048 l gs col0 s gr % Polyline n 12267 6858 m 12341 6858 l gs col1 s gr % Polyline n 11232 7523 m 12304 6858 l 13375 6475 l 14447 6783 l 15518 6700 l 16590 6912 l gs col1 s gr % Polyline n 4938 7271 m 5012 7197 l gs col3 s gr % Polyline n 16553 6366 m 16590 6403 l 16627 6366 l 16590 6329 l 16553 6366 l cp gs col0 s gr % Polyline n 15514 6666 m 15522 6666 l gs col0 s gr % Polyline n 15481 6666 m 15518 6703 l 15555 6666 l 15518 6629 l 15481 6666 l cp gs col0 s gr % Polyline n 14443 6639 m 14451 6639 l gs col0 s gr % Polyline n 14410 6639 m 14447 6676 l 14484 6639 l 14447 6602 l 14410 6639 l cp gs col0 s gr % Polyline [15 60] 60 sd n 2832 8977 m 8190 8977 l gs col0 s gr [] 0 sd % Polyline n 8190 8977 m 8115 8977 l gs col0 s gr % Polyline n 8190 8443 m 8115 8443 l gs col0 s gr % Polyline n 8190 7909 m 8115 7909 l gs col0 s gr % Polyline n 8190 7375 m 8115 7375 l gs col0 s gr % Polyline n 8190 6841 m 8115 6841 l gs col0 s gr % Polyline [15 60] 60 sd n 8116 6307 m 8190 6307 l gs col0 s gr [] 0 sd % Polyline n 8190 6307 m 8115 6307 l gs col0 s gr % Polyline [15 60] 60 sd n 2832 5773 m 8190 5773 l gs col0 s gr [] 0 sd % Polyline n 2832 8145 m 3904 7770 l 4975 7366 l 6047 7285 l 7118 7371 l 8190 6524 l gs col4 s gr % Polyline n 3904 8977 m 3904 8902 l gs col0 s gr % Polyline n 3904 5773 m 3904 5848 l gs col0 s gr % Polyline n 4975 8977 m 4975 8902 l gs col0 s gr % Polyline n 4975 5773 m 4975 5848 l gs col0 s gr % Polyline n 6047 8977 m 6047 8902 l gs col0 s gr % Polyline n 6047 5773 m 6047 5848 l gs col0 s gr % Polyline n 7118 8977 m 7118 8902 l gs col0 s gr % Polyline n 7118 5773 m 7118 5848 l gs col0 s gr % Polyline [15 60] 60 sd n 8190 8977 m 8190 5773 l gs col0 s gr [] 0 sd % Polyline n 8190 8977 m 8190 8902 l gs col0 s gr % Polyline n 8190 5773 m 8190 5848 l gs col0 s gr % Polyline n 2832 8145 m 3904 7771 l 4975 7553 l 6047 6543 l 7118 6621 l 8190 6061 l gs col0 s gr % Polyline n 3867 7771 m 3904 7808 l 3941 7771 l 3904 7734 l 3867 7771 l cp gs col0 s gr % Polyline n 3900 7771 m 3908 7771 l gs col0 s gr % Polyline n 4938 7553 m 4975 7590 l 5012 7553 l 4975 7516 l 4938 7553 l cp gs col0 s gr % Polyline n 4971 7553 m 4979 7553 l gs col0 s gr % Polyline n 6010 6543 m 6047 6580 l 6084 6543 l 6047 6506 l 6010 6543 l cp gs col0 s gr % Polyline n 6043 6543 m 6051 6543 l gs col0 s gr % Polyline n 7081 6621 m 7118 6658 l 7155 6621 l 7118 6584 l 7081 6621 l cp gs col0 s gr % Polyline n 7114 6621 m 7122 6621 l gs col0 s gr % Polyline n 8153 6061 m 8190 6098 l 8227 6061 l 8190 6024 l 8153 6061 l cp gs col0 s gr % Polyline n 8186 6061 m 8194 6061 l gs col0 s gr % Polyline n 2832 8145 m 3904 7839 l 4975 7305 l 6047 6722 l 7118 6435 l 8190 6862 l gs col1 s gr % Polyline n 3867 7839 m 3941 7839 l gs col1 s gr % Polyline n 3904 7876 m 3904 7802 l gs col1 s gr % Polyline n 4938 7305 m 5012 7305 l gs col1 s gr % Polyline n 4975 7342 m 4975 7268 l gs col1 s gr % Polyline n 6010 6722 m 6084 6722 l gs col1 s gr % Polyline n 6047 6759 m 6047 6685 l gs col1 s gr % Polyline n 7114 7371 m 7122 7371 l gs col4 s gr % Polyline n 13338 7446 m 13375 7483 l 13412 7446 l 13375 7409 l 13338 7446 l cp gs col0 s gr % Polyline n 8153 6862 m 8227 6862 l gs col1 s gr % Polyline n 8190 6899 m 8190 6825 l gs col1 s gr % Polyline n 2832 8145 m 3904 8033 l 4975 7235 l 6047 7239 l 7118 7673 l 8190 6076 l gs col2 s gr % Polyline n 3867 8070 m 3941 8070 l 3941 7996 l 3867 7996 l 3867 8070 l cp gs col2 s gr % Polyline n 3900 8033 m 3908 8033 l gs col2 s gr % Polyline n 4938 7272 m 5012 7272 l 5012 7198 l 4938 7198 l 4938 7272 l cp gs col2 s gr % Polyline n 4971 7235 m 4979 7235 l gs col2 s gr % Polyline n 6010 7276 m 6084 7276 l 6084 7202 l 6010 7202 l 6010 7276 l cp gs col2 s gr % Polyline n 6043 7239 m 6051 7239 l gs col2 s gr % Polyline n 7081 7710 m 7155 7710 l 7155 7636 l 7081 7636 l 7081 7710 l cp gs col2 s gr % Polyline n 7114 7673 m 7122 7673 l gs col2 s gr % Polyline n 8153 6113 m 8227 6113 l 8227 6039 l 8153 6039 l 8153 6113 l cp gs col2 s gr % Polyline n 8186 6076 m 8194 6076 l gs col2 s gr % Polyline n 2832 8145 m 3904 7838 l 4975 7234 l 6047 7129 l 7118 6992 l 8190 7093 l gs col3 s gr % Polyline n 3867 7875 m 3941 7801 l gs col3 s gr % Polyline n 3867 7801 m 3941 7875 l gs col3 s gr % Polyline n 6010 7496 m 6084 7570 l gs col5 s gr % Polyline n 4938 7197 m 5012 7271 l gs col3 s gr % Polyline n 6010 7166 m 6084 7092 l gs col3 s gr % Polyline n 6010 7092 m 6084 7166 l gs col3 s gr % Polyline n 7081 7029 m 7155 6955 l gs col3 s gr % Polyline n 7081 6955 m 7155 7029 l gs col3 s gr % Polyline n 8153 7130 m 8227 7056 l gs col3 s gr % Polyline n 8153 7056 m 8227 7130 l gs col3 s gr % Polyline n 2832 8145 m 3904 7677 l 4975 6979 l 6047 7156 l 7118 6890 l 8190 6759 l gs col6 s gr % Polyline n 3904 7721 m 3855 7794 l 3953 7794 l 3904 7721 l cp gs col4 s gr % Polyline n 3900 7770 m 3908 7770 l gs col4 s gr % Polyline n 4975 7317 m 4926 7390 l 5024 7390 l 4975 7317 l cp gs col4 s gr % Polyline n 4971 7366 m 4979 7366 l gs col4 s gr % Polyline n 1500 8977 m 8190 8977 l 8190 5773 l 1500 5773 l 1500 8977 l cp gs col0 s gr % Polyline n 6043 7285 m 6051 7285 l gs col4 s gr % Polyline n 7118 7322 m 7069 7395 l 7167 7395 l 7118 7322 l cp gs col4 s gr % Polyline n 4971 6979 m 4979 6979 l gs col6 s gr % Polyline n 8190 6475 m 8141 6548 l 8239 6548 l 8190 6475 l cp gs col4 s gr % Polyline n 8186 6524 m 8194 6524 l gs col4 s gr % Polyline n 2832 8145 m 3904 7635 l 4975 7467 l 6047 7533 l 7118 7056 l 8190 6700 l gs col5 s gr % Polyline n 3867 7635 m 3941 7635 l gs col5 s gr % Polyline n 3904 7672 m 3904 7598 l gs col5 s gr % Polyline n 3867 7672 m 3941 7598 l gs col5 s gr % Polyline n 3867 7598 m 3941 7672 l gs col5 s gr % Polyline n 4938 7467 m 5012 7467 l gs col5 s gr % Polyline n 4975 7504 m 4975 7430 l gs col5 s gr % Polyline n 4938 7504 m 5012 7430 l gs col5 s gr % Polyline n 4938 7430 m 5012 7504 l gs col5 s gr % Polyline n 6010 7533 m 6084 7533 l gs col5 s gr % Polyline n 6047 7570 m 6047 7496 l gs col5 s gr % Polyline n 6010 7570 m 6084 7496 l gs col5 s gr % Polyline n 7114 6890 m 7122 6890 l gs col6 s gr % Polyline n 7081 7056 m 7155 7056 l gs col5 s gr % Polyline n 7118 7093 m 7118 7019 l gs col5 s gr % Polyline n 7081 7093 m 7155 7019 l gs col5 s gr % Polyline n 7081 7019 m 7155 7093 l gs col5 s gr % Polyline n 8153 6700 m 8227 6700 l gs col5 s gr % Polyline n 8190 6737 m 8190 6663 l gs col5 s gr % Polyline n 8153 6737 m 8227 6663 l gs col5 s gr % Polyline n 8153 6663 m 8227 6737 l gs col5 s gr % Polyline n 16590 5773 m 16590 5848 l gs col0 s gr % Polyline n 3867 7677 m 3904 7714 l 3941 7677 l 3904 7640 l 3867 7677 l cp gs col6 s gr % Polyline n 3900 7677 m 3908 7677 l gs col6 s gr % Polyline n 4938 6979 m 4975 7016 l 5012 6979 l 4975 6942 l 4938 6979 l cp gs col6 s gr % Polyline [15 60] 60 sd n 11232 8977 m 16590 8977 l gs col0 s gr [] 0 sd % Polyline n 6010 7156 m 6047 7193 l 6084 7156 l 6047 7119 l 6010 7156 l cp gs col6 s gr % Polyline n 6043 7156 m 6051 7156 l gs col6 s gr % Polyline n 7081 6890 m 7118 6927 l 7155 6890 l 7118 6853 l 7081 6890 l cp gs col6 s gr % Polyline n 16590 8336 m 16515 8336 l gs col0 s gr % Polyline n 8153 6759 m 8190 6796 l 8227 6759 l 8190 6722 l 8153 6759 l cp gs col6 s gr % Polyline n 8186 6759 m 8194 6759 l gs col6 s gr % Polyline n 12300 7173 m 12308 7173 l gs col0 s gr % Polyline n 12267 7173 m 12304 7210 l 12341 7173 l 12304 7136 l 12267 7173 l cp gs col0 s gr % Polyline n 11232 7523 m 12304 7173 l 13375 7446 l 14447 6639 l 15518 6666 l 16590 6366 l gs col0 s gr % Polyline n 9900 8977 m 16590 8977 l 16590 5773 l 9900 5773 l 9900 8977 l cp gs col0 s gr % Polyline n 16590 7055 m 16515 7055 l gs col0 s gr % Polyline n 8190 4177 m 8115 4177 l gs col0 s gr % Polyline [15 60] 60 sd n 2832 4177 m 8190 4177 l gs col0 s gr [] 0 sd % Polyline n 8190 4177 m 8190 4102 l gs col0 s gr % Polyline n 1500 4177 m 8190 4177 l 8190 973 l 1500 975 l 1500 4177 l cp gs col0 s gr % Polyline [15 60] 60 sd n 11232 5773 m 16590 5773 l gs col0 s gr [] 0 sd % Polyline n 16590 8977 m 16515 8977 l gs col0 s gr % Polyline n 12304 5773 m 12304 5848 l gs col0 s gr % Polyline n 16590 7695 m 16515 7695 l gs col0 s gr % Polyline n 13375 8977 m 13375 8902 l gs col0 s gr % Polyline [15 60] 60 sd n 16516 6414 m 16590 6414 l gs col0 s gr [] 0 sd % Polyline n 16590 6414 m 16515 6414 l gs col0 s gr % Polyline n 13375 5773 m 13375 5848 l gs col0 s gr % Polyline n 16590 5773 m 16515 5773 l gs col0 s gr % Polyline n 12304 8977 m 12304 8902 l gs col0 s gr % Polyline n 14447 8977 m 14447 8902 l gs col0 s gr % Polyline n 14447 5773 m 14447 5848 l gs col0 s gr % Polyline n 15518 8977 m 15518 8902 l gs col0 s gr % Polyline n 15518 5773 m 15518 5848 l gs col0 s gr % Polyline [15 60] 60 sd n 16590 8977 m 16590 5773 l gs col0 s gr [] 0 sd % Polyline n 16590 8977 m 16590 8902 l gs col0 s gr % Polyline n 10425 1275 m 13800 1275 l 13800 1950 l 10425 1950 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 12675 7575 m 16275 7575 l 16275 8250 l 12675 8250 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 2025 6075 m 5625 6075 l 5625 6750 l 2025 6750 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 2025 1275 m 5625 1275 l 5625 1950 l 2025 1950 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 3171 1483 m 3542 1483 l gs col2 s gr % Polyline n 3319 1520 m 3393 1520 l 3393 1446 l 3319 1446 l 3319 1520 l cp gs col2 s gr % Polyline n 3352 1483 m 3360 1483 l gs col2 s gr /Times-Bold ff 240.00 scf sf 3097 1545 m gs 1 -1 sc (X) dup sw pop neg 0 rm col2 sh gr % Polyline n 2346 1485 m 2717 1485 l gs col0 s gr % Polyline n 5269 1479 m 5306 1516 l 5343 1479 l 5306 1442 l 5269 1479 l cp gs col6 s gr % Polyline n 2494 1485 m 2531 1522 l 2568 1485 l 2531 1448 l 2494 1485 l cp gs col0 s gr % Polyline n 2527 1485 m 2535 1485 l gs col0 s gr /Times-Bold ff 240.00 scf sf 2272 1547 m gs 1 -1 sc (Z) dup sw pop neg 0 rm col0 sh gr % Polyline n 2346 1759 m 2717 1759 l gs col1 s gr % Polyline n 2494 1759 m 2568 1759 l gs col1 s gr % Polyline n 2531 1796 m 2531 1722 l gs col1 s gr /Times-Bold ff 240.00 scf sf 2272 1821 m gs 1 -1 sc (Y) dup sw pop neg 0 rm col1 sh gr % Polyline n 3171 1757 m 3542 1757 l gs col3 s gr % Polyline n 3319 1794 m 3393 1720 l gs col3 s gr % Polyline n 3319 1720 m 3393 1794 l gs col3 s gr /Times-Bold ff 240.00 scf sf 3097 1819 m gs 1 -1 sc (ZY) dup sw pop neg 0 rm col3 sh gr % Polyline n 3996 1481 m 4367 1481 l gs col4 s gr % Polyline n 4181 1432 m 4132 1505 l 4230 1505 l 4181 1432 l cp gs col4 s gr % Polyline n 4177 1481 m 4185 1481 l gs col4 s gr /Times-Bold ff 240.00 scf sf 3922 1543 m gs 1 -1 sc (ZX) dup sw pop neg 0 rm col4 sh gr % Polyline n 3996 1755 m 4367 1755 l gs col5 s gr % Polyline n 4144 1755 m 4218 1755 l gs col5 s gr % Polyline n 4181 1792 m 4181 1718 l gs col5 s gr % Polyline n 4144 1792 m 4218 1718 l gs col5 s gr % Polyline n 4144 1718 m 4218 1792 l gs col5 s gr /Times-Bold ff 240.00 scf sf 3922 1817 m gs 1 -1 sc (YX) dup sw pop neg 0 rm col5 sh gr /Times-Bold ff 240.00 scf sf 14647 8117 m gs 1 -1 sc (YX) dup sw pop neg 0 rm col5 sh gr % Polyline n 14869 8018 m 14943 8092 l gs col5 s gr % Polyline n 14869 8092 m 14943 8018 l gs col5 s gr % Polyline n 14906 8092 m 14906 8018 l gs col5 s gr % Polyline n 14869 8055 m 14943 8055 l gs col5 s gr % Polyline n 14721 8055 m 15092 8055 l gs col5 s gr /Times-Bold ff 240.00 scf sf 14647 7843 m gs 1 -1 sc (ZX) dup sw pop neg 0 rm col4 sh gr % Polyline n 14902 7781 m 14910 7781 l gs col4 s gr % Polyline n 14906 7732 m 14857 7805 l 14955 7805 l 14906 7732 l cp gs col4 s gr % Polyline n 14721 7781 m 15092 7781 l gs col4 s gr /Times-Bold ff 240.00 scf sf 13822 8119 m gs 1 -1 sc (ZY) dup sw pop neg 0 rm col3 sh gr % Polyline n 14044 8020 m 14118 8094 l gs col3 s gr % Polyline n 14044 8094 m 14118 8020 l gs col3 s gr % Polyline n 13896 8057 m 14267 8057 l gs col3 s gr /Times-Bold ff 240.00 scf sf 12997 8121 m gs 1 -1 sc (Y) dup sw pop neg 0 rm col1 sh gr % Polyline n 13256 8096 m 13256 8022 l gs col1 s gr % Polyline n 13219 8059 m 13293 8059 l gs col1 s gr % Polyline n 13071 8059 m 13442 8059 l gs col1 s gr /Times-Bold ff 240.00 scf sf 12997 7847 m gs 1 -1 sc (Z) dup sw pop neg 0 rm col0 sh gr % Polyline n 13252 7785 m 13260 7785 l gs col0 s gr % Polyline n 13219 7785 m 13256 7822 l 13293 7785 l 13256 7748 l 13219 7785 l cp gs col0 s gr % Polyline n 13071 7785 m 13442 7785 l gs col0 s gr /Times-Bold ff 240.00 scf sf 13822 7845 m gs 1 -1 sc (X) dup sw pop neg 0 rm col2 sh gr % Polyline n 14077 7783 m 14085 7783 l gs col2 s gr % Polyline n 14044 7820 m 14118 7820 l 14118 7746 l 14044 7746 l 14044 7820 l cp gs col2 s gr % Polyline n 13896 7783 m 14267 7783 l gs col2 s gr /Times-Bold ff 240.00 scf sf 15697 7841 m gs 1 -1 sc (ZYX) dup sw pop neg 0 rm col6 sh gr % Polyline n 15952 7779 m 15960 7779 l gs col6 s gr % Polyline n 15919 7779 m 15956 7816 l 15993 7779 l 15956 7742 l 15919 7779 l cp gs col6 s gr % Polyline n 15771 7779 m 16142 7779 l gs col6 s gr /Times-Bold ff 270.00 scf sf 10500 8175 m gs 1 -1 sc (netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 10500 7875 m gs 1 -1 sc (Serial) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 2100 3150 m gs 1 -1 sc (netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 2100 2850 m gs 1 -1 sc (Serial) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 10500 3525 m gs 1 -1 sc (netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 10500 3225 m gs 1 -1 sc (Serial) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 240.00 scf sf 12322 1817 m gs 1 -1 sc (YX) dup sw pop neg 0 rm col5 sh gr % Polyline n 12544 1718 m 12618 1792 l gs col5 s gr % Polyline n 12544 1792 m 12618 1718 l gs col5 s gr % Polyline n 12581 1792 m 12581 1718 l gs col5 s gr % Polyline n 12544 1755 m 12618 1755 l gs col5 s gr % Polyline n 12396 1755 m 12767 1755 l gs col5 s gr /Times-Bold ff 240.00 scf sf 12322 1543 m gs 1 -1 sc (ZX) dup sw pop neg 0 rm col4 sh gr % Polyline n 12577 1481 m 12585 1481 l gs col4 s gr % Polyline n 12581 1432 m 12532 1505 l 12630 1505 l 12581 1432 l cp gs col4 s gr % Polyline n 12396 1481 m 12767 1481 l gs col4 s gr /Times-Bold ff 240.00 scf sf 11497 1819 m gs 1 -1 sc (ZY) dup sw pop neg 0 rm col3 sh gr % Polyline n 11719 1720 m 11793 1794 l gs col3 s gr % Polyline n 11719 1794 m 11793 1720 l gs col3 s gr % Polyline n 11571 1757 m 11942 1757 l gs col3 s gr /Times-Bold ff 240.00 scf sf 10672 1821 m gs 1 -1 sc (Y) dup sw pop neg 0 rm col1 sh gr % Polyline n 10931 1796 m 10931 1722 l gs col1 s gr % Polyline n 10894 1759 m 10968 1759 l gs col1 s gr % Polyline n 10746 1759 m 11117 1759 l gs col1 s gr /Times-Bold ff 240.00 scf sf 10672 1547 m gs 1 -1 sc (Z) dup sw pop neg 0 rm col0 sh gr % Polyline n 10927 1485 m 10935 1485 l gs col0 s gr % Polyline n 10894 1485 m 10931 1522 l 10968 1485 l 10931 1448 l 10894 1485 l cp gs col0 s gr % Polyline n 10746 1485 m 11117 1485 l gs col0 s gr /Times-Bold ff 240.00 scf sf 11497 1545 m gs 1 -1 sc (X) dup sw pop neg 0 rm col2 sh gr % Polyline n 11752 1483 m 11760 1483 l gs col2 s gr % Polyline n 11719 1520 m 11793 1520 l 11793 1446 l 11719 1446 l 11719 1520 l cp gs col2 s gr % Polyline n 11571 1483 m 11942 1483 l gs col2 s gr /Times-Bold ff 240.00 scf sf 13222 1541 m gs 1 -1 sc (ZYX) dup sw pop neg 0 rm col6 sh gr % Polyline n 13477 1479 m 13485 1479 l gs col6 s gr % Polyline n 13444 1479 m 13481 1516 l 13518 1479 l 13481 1442 l 13444 1479 l cp gs col6 s gr % Polyline n 13296 1479 m 13667 1479 l gs col6 s gr /Times-Bold ff 240.00 scf sf 3997 6617 m gs 1 -1 sc (YX) dup sw pop neg 0 rm col5 sh gr % Polyline n 4219 6518 m 4293 6592 l gs col5 s gr % Polyline n 4219 6592 m 4293 6518 l gs col5 s gr % Polyline n 4256 6592 m 4256 6518 l gs col5 s gr % Polyline n 4219 6555 m 4293 6555 l gs col5 s gr % Polyline n 4071 6555 m 4442 6555 l gs col5 s gr /Times-Bold ff 240.00 scf sf 3997 6343 m gs 1 -1 sc (ZX) dup sw pop neg 0 rm col4 sh gr % Polyline n 4252 6281 m 4260 6281 l gs col4 s gr % Polyline n 4256 6232 m 4207 6305 l 4305 6305 l 4256 6232 l cp gs col4 s gr % Polyline n 4071 6281 m 4442 6281 l gs col4 s gr /Times-Bold ff 240.00 scf sf 3172 6619 m gs 1 -1 sc (ZY) dup sw pop neg 0 rm col3 sh gr % Polyline n 3394 6520 m 3468 6594 l gs col3 s gr % Polyline n 3394 6594 m 3468 6520 l gs col3 s gr % Polyline n 3246 6557 m 3617 6557 l gs col3 s gr /Times-Bold ff 240.00 scf sf 2347 6621 m gs 1 -1 sc (Y) dup sw pop neg 0 rm col1 sh gr % Polyline n 2606 6596 m 2606 6522 l gs col1 s gr % Polyline n 2569 6559 m 2643 6559 l gs col1 s gr % Polyline n 2421 6559 m 2792 6559 l gs col1 s gr /Times-Bold ff 240.00 scf sf 2347 6347 m gs 1 -1 sc (Z) dup sw pop neg 0 rm col0 sh gr % Polyline n 2602 6285 m 2610 6285 l gs col0 s gr % Polyline n 2569 6285 m 2606 6322 l 2643 6285 l 2606 6248 l 2569 6285 l cp gs col0 s gr % Polyline n 2421 6285 m 2792 6285 l gs col0 s gr /Times-Bold ff 240.00 scf sf 3172 6345 m gs 1 -1 sc (X) dup sw pop neg 0 rm col2 sh gr % Polyline n 3427 6283 m 3435 6283 l gs col2 s gr % Polyline n 3394 6320 m 3468 6320 l 3468 6246 l 3394 6246 l 3394 6320 l cp gs col2 s gr % Polyline n 3246 6283 m 3617 6283 l gs col2 s gr /Times-Bold ff 240.00 scf sf 5047 6341 m gs 1 -1 sc (ZYX) dup sw pop neg 0 rm col6 sh gr % Polyline n 5302 6279 m 5310 6279 l gs col6 s gr % Polyline n 5269 6279 m 5306 6316 l 5343 6279 l 5306 6242 l 5269 6279 l cp gs col6 s gr % Polyline n 5121 6279 m 5492 6279 l gs col6 s gr % Polyline n 5302 1479 m 5310 1479 l gs col6 s gr /Times-Bold ff 240.00 scf sf 5047 1541 m gs 1 -1 sc (ZYX) dup sw pop neg 0 rm col6 sh gr % Polyline n 5121 1479 m 5492 1479 l gs col6 s gr /Times-Bold ff 360.00 scf sf 4725 5550 m gs 1 -1 sc (Read 1 GB) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 13350 750 m gs 1 -1 sc (Write 64 MB) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 4725 750 m gs 1 -1 sc (Read 64 MB) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 4800 4800 m gs 1 -1 sc (Number of processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 13200 4800 m gs 1 -1 sc (Number of processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 13200 9600 m gs 1 -1 sc (Number of processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 4800 9600 m gs 1 -1 sc (Number of processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 9150 7350 m gs 1 -1 sc 90.0 rot (Bandwidth \(MB/sec\)) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 750 7350 m gs 1 -1 sc 90.0 rot (Bandwidth \(MB/sec\)) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 9150 2550 m gs 1 -1 sc 90.0 rot (Bandwidth \(MB/sec\)) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 750 2475 m gs 1 -1 sc 90.0 rot (Bandwidth \(MB/sec\)) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 360.00 scf sf 13350 5550 m gs 1 -1 sc (Write 1 GB) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 2100 7500 m gs 1 -1 sc (Serial) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 270.00 scf sf 2100 7800 m gs 1 -1 sc (netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Bold ff 300.00 scf sf 12572 4438 m gs 1 -1 sc (2) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 1035 m gs 1 -1 sc (700) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 8190 9238 m gs 1 -1 sc (32) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 9039 m gs 1 -1 sc (0) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 8398 m gs 1 -1 sc (50) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 16590 4438 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 13911 4438 m gs 1 -1 sc (4) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 6476 m gs 1 -1 sc (200) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 1493 m gs 1 -1 sc (600) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 1950 m gs 1 -1 sc (500) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 2408 m gs 1 -1 sc (400) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 2866 m gs 1 -1 sc (300) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 3324 m gs 1 -1 sc (200) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 15251 4438 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 7757 m gs 1 -1 sc (100) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 7117 m gs 1 -1 sc (150) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9808 5835 m gs 1 -1 sc (250) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 3781 m gs 1 -1 sc (100) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 5511 4438 m gs 1 -1 sc (4) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 11231 9238 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 15525 9225 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 13345 9238 m gs 1 -1 sc (4) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 12288 9238 m gs 1 -1 sc (2) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 5835 m gs 1 -1 sc (300) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 6369 m gs 1 -1 sc (250) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 6903 m gs 1 -1 sc (200) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 7437 m gs 1 -1 sc (150) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 7971 m gs 1 -1 sc (100) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 8505 m gs 1 -1 sc (50) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 9039 m gs 1 -1 sc (0) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 11232 4438 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 7118 9238 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9733 4239 m gs 1 -1 sc (0) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 6047 9238 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 4975 9238 m gs 1 -1 sc (4) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 3904 9238 m gs 1 -1 sc (2) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 2832 9238 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9733 1035 m gs 1 -1 sc (200) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9733 1836 m gs 1 -1 sc (150) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9733 2637 m gs 1 -1 sc (100) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1333 4239 m gs 1 -1 sc (0) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9900 9238 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 8190 4438 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9733 3438 m gs 1 -1 sc (50) dup sw pop neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 6851 4438 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 4172 4438 m gs 1 -1 sc (2) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 2832 4438 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 14475 9225 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 16575 9225 m gs 1 -1 sc (32) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1500 9238 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 1500 4438 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Bold ff 300.00 scf sf 9900 4438 m gs 1 -1 sc (1) dup sw pop 2 div neg 0 rm col0 sh gr $F2psEnd rs %%EndDocument @endspecial -14 2306 a Fe(Figure)25 b(6.)361 2207 y SDict begin H.S end 361 2207 a 361 2207 a SDict begin H.R end 361 2207 a 361 2207 a SDict begin [ /View [/XYZ H.V] /Dest (figure.6) cvn H.B /DEST pdfmark end 361 2207 a 99 x Fe(Serial)i(and)e(parallel)i(netCDF)e(perf)n(ormance)h(f)n(or)g (64)h(MB)f(and)f(1)h(GB)g(datasets.)41 b(The)25 b(\002r)o(st)h(column)f (of)-14 2406 y(eac)o(h)d(c)o(har)r(t)g(sho)o(ws)f(the)g(I/O)h(perf)n (ormance)g(of)g(reading/writing)f(the)g(whole)g(arra)n(y)j(thr)n(ough)d (a)h(single)g(pr)n(ocessor)-14 2506 y(using)g(serial)i(netCDF;)e(the)h (rest)g(of)f(the)h(columns)f(sho)o(w)h(the)f(results)h(using)g (parallel)h(netCDF)-8 b(.)-114 2856 y Fg(I/O)33 b(when)g(using)g(lar)o (ge)f(\002le)i(size.)65 b(The)33 b(I/O)g(bandwidth)f(does)-114 2955 y(not)g(scale)h(in)f(direct)g(proportion)d(because)j(the)g(number) e(of)i(I/O)-114 3055 y(nodes)c(\(and)f(disks\))i(is)g(\002x)o(ed)f(so)h (that)g(the)f(dominating)f(disk)h(ac-)-114 3155 y(cess)19 b(time)g(at)g(I/O)g(nodes)e(is)j(almost)e(\002x)o(ed.)24 b(As)19 b(e)o(xpected,)e(the)i(par)n(-)-114 3254 y(allel)k(netCDF)g (outperforms)d(the)i(original)f(serial)i(netCDF)g(as)g(the)-114 3354 y(number)d(of)h(processes)g(increases.)28 b(The)21 b(dif)n(ference)f(between)g(the)-114 3453 y(serial)g(netCDF)h (performance)c(and)j(the)g(parallel)f(netCDF)i(perfor)n(-)-114 3553 y(mance)16 b(with)h(single)f(processor)f(is)j(because)e(of)g (their)g(dif)n(ferent)f(I/O)-114 3653 y(implementations)k(and)h(dif)n (ferent)f(I/O)i(caching/b)n(uf)n(fering)c(strate-)-114 3752 y(gies.)31 b(In)21 b(the)h(serial)h(netCDF)f(case,)h(if,)f(as)h (in)f(Figure)1474 3752 y SDict begin H.S end 1474 3752 a Fg(2)1516 3696 y SDict begin H.R end 1516 3696 a 1516 3752 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.2) cvn H.B /ANN pdfmark end 1516 3752 a Fg(\(a\),)g(multi-)-114 3852 y(processors)29 b(were)h(used)g(and)g(the)g(R)m(OO)m(T)g (processor)f(needed)g(to)-114 3952 y(collect)21 b(partitioned)f(data)h (and)g(then)g(perform)e(the)i(serial)h(netCDF)-114 4051 y(I/O,)30 b(the)g(performance)d(w)o(ould)i(be)h(much)f(w)o(orse)h(and)f (decrease)-114 4151 y(with)35 b(the)f(number)f(of)h(processors)g (because)g(of)g(the)h(additional)-114 4250 y(communication)17 b(cost)j(and)f(di)n(vision)g(of)g(a)i(lar)o(ge)d(I/O)i(request)g(into) -114 4350 y(a)h(series)f(of)g(small)h(requests.)-114 4491 y SDict begin H.S end -114 4491 a -114 4491 a SDict begin 12 H.A end -114 4491 a -114 4491 a SDict begin [ /View [/XYZ H.V] /Dest (subsection.5.2) cvn H.B /DEST pdfmark end -114 4491 a 97 x Ff(5.2)90 b(FLASH)23 b(I/O)g(P)n(erf)n(ormance)-14 4827 y Fg(The)d(FLASH)i(I/O)f(benchmark)d(simulates)j(the)g(I/O)h (pattern)e(of)-114 4926 y(an)33 b(important)f(scienti\002c)i (application)e(called)h(FLASH)h([)1649 4926 y SDict begin H.S end 1649 4926 a Fg(1)1690 4870 y SDict begin H.R end 1690 4870 a 1690 4926 a SDict begin [ /Color [0 1 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (cite.FORT00) cvn H.B /ANN pdfmark end 1690 4926 a Fg(].)65 b(It)-114 5026 y(recreates)32 b(the)g(primary)f(data)h (structures)g(in)g(the)g(FLASH)h(code)-114 5126 y(and)17 b(produces)e(a)j(checkpoint)d(\002le,)j(a)g(plot\002le)f(with)h (centered)e(data,)-114 5225 y(and)23 b(a)g(plot\002le)g(with)h(corner)d (data,)j(using)f(parallel)f(HDF5.)34 b(Basi-)-114 5325 y(cally)-5 b(,)30 b(these)f(three)f(output)g(\002les)h(contains)f(a)h (series)h(of)e(multidi-)2042 2856 y(mensional)d(arrays,)i(and)f(the)g (access)h(pattern)f(is)h(simple)f(\(Block,)2042 2955 y(*,)33 b(...\),)e(which)f(is)h(similar)f(to)g(the)g(Z)g(partition)f (in)h(Figure)3824 2956 y SDict begin H.S end 3824 2956 a -1 x Fg(5)3866 2899 y SDict begin H.R end 3866 2899 a 3866 2955 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.5) cvn H.B /ANN pdfmark end 3866 2955 a Fg(.)55 b(In)2042 3055 y(each)31 b(of)f(the)h(\002les,)j(the)c(benchmark)e(writes)j(the)g (related)f(arrays)2042 3155 y(in)24 b(a)h(\002x)o(ed)e(order)g(from)g (contiguous)f(user)i(b)n(uf)n(fers,)g(respecti)n(v)o(ely)-5 b(.)2042 3254 y(The)23 b(I/O)g(routines)f(in)h(the)g(benchmark)d(are)j (identical)g(to)g(the)g(rou-)2042 3354 y(tines)30 b(used)f(by)g(FLASH,) g(so)h(an)o(y)e(performance)e(impro)o(v)o(ements)2042 3453 y(made)19 b(to)h(the)g(benchmark)d(program)g(will)k(be)e(shared)g (by)g(FLASH.)2042 3553 y(In)28 b(our)f(e)o(xperiments,)g(in)h(order)f (to)h(focus)f(on)g(the)h(data)g(I/O)g(per)n(-)2042 3653 y(formance,)i(we)g(modi\002ed)e(this)i(benchmark,)f(remo)o(v)o(ed)d (the)k(part)2042 3752 y(of)22 b(code)g(writing)g(attrib)n(utes,)g (ported)f(it)i(to)f(parallel)g(netCDF)-7 b(,)23 b(and)2042 3852 y(observ)o(ed)18 b(the)j(ef)n(fect)e(of)h(our)g(ne)n(w)g(parallel) f(I/O)i(approach.)2142 4030 y(Figure)2388 4031 y SDict begin H.S end 2388 4031 a -1 x Fg(7)2429 3974 y SDict begin H.R end 2429 3974 a 2429 4030 a SDict begin [ /Color [1 0 0] /H /I /Border [0 0 12] /Subtype /Link /Dest (figure.7) cvn H.B /ANN pdfmark end 2429 4030 a 30 w Fg(sho)n(ws)28 b(the)h(performance)d(results)j(of)f (the)h(FLASH)2042 4129 y(I/O)g(benchmark)c(using)j(parallel)f(netCDF)i (and)e(parallel)h(HDF5.)2042 4229 y(W)-7 b(e)24 b(tested)f(both)e (small)i(data)g(size)g(and)f(lar)o(ge)g(data)g(size.)33 b(The)22 b(pa-)2042 4329 y(rameters)i(used)g(in)g(these)g(tw)o(o)h(e)o (xperiments)d(are:)33 b(\(a\))24 b(nxb)f(=)i(n)o(yb)2042 4428 y(=)30 b(nzb)g(=)g(8,)i(nguard)c(=)i(4,)i(number)c(of)h(blocks)g (=)h(80,)i(and)d(n)m(v)n(ar)2042 4528 y(=)i(24;)k(\(b\))30 b(nxb)g(=)h(n)o(yb)e(=)i(nzb)f(=)h(16,)h(nguard)c(=)j(8,)i(number)c(of) 2042 4628 y(blocks)i(=)g(80,)i(and)d(n)m(v)n(ar)g(=)h(24.)57 b(Although)30 b(both)g(I/O)h(libraries)2042 4727 y(are)22 b(b)n(uilt)h(abo)o(v)o(e)d(MPI-IO,)h(the)i(parallel)e(netCDF)i(has)f (much)f(less)2042 4827 y(o)o(v)o(erhead)30 b(and)h(outperforms)e (parallel)j(HDF5)g(by)g(almost)g(dou-)2042 4926 y(bling)27 b(the)g(o)o(v)o(erall)f(I/O)h(rate.)46 b(The)26 b(e)o(xtra)h(o)o(v)o (erhead)d(in)m(v)n(olv)o(ed)h(in)2042 5026 y(the)17 b(current)e (release)h(of)h(HDF5)f(\(v)o(ersion)f(5-1.4.3\))f(includes)i(inter)n(-) 2042 5126 y(process)25 b(synchronizations)d(and)i(\002le)h(header)f (access)h(performed)2042 5225 y(internally)18 b(in)h(parallel)g (open/close)e(of)i(e)n(v)o(ery)f(dataset)h(\(analogous)2042 5325 y(to)24 b(a)f(netCDF)h(v)n(ariable\))e(and)h(recursi)n(v)o(e)f (handling)f(of)i(the)g(hyper)n(-)1928 5574 y(8)p eop %%Page: 9 9 9 8 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.9) cvn H.B /DEST pdfmark end -114 0 a 92 1383 a @beginspecial 0 @llx 0 @lly 738 @urx 275 @ury 4455 @rwi @setspecial %%BeginDocument: flashio.eps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /col-1 {0 setgray} bind def /col0 {0.000 0.000 0.000 srgb} bind def /col1 {0.000 0.000 1.000 srgb} bind def /col2 {0.000 1.000 0.000 srgb} bind def /col3 {0.000 1.000 1.000 srgb} bind def /col4 {1.000 0.000 0.000 srgb} bind def /col5 {1.000 0.000 1.000 srgb} bind def /col6 {1.000 1.000 0.000 srgb} bind def /col7 {1.000 1.000 1.000 srgb} bind def /col8 {0.000 0.000 0.560 srgb} bind def /col9 {0.000 0.000 0.690 srgb} bind def /col10 {0.000 0.000 0.820 srgb} bind def /col11 {0.530 0.810 1.000 srgb} bind def /col12 {0.000 0.560 0.000 srgb} bind def /col13 {0.000 0.690 0.000 srgb} bind def /col14 {0.000 0.820 0.000 srgb} bind def /col15 {0.000 0.560 0.560 srgb} bind def /col16 {0.000 0.690 0.690 srgb} bind def /col17 {0.000 0.820 0.820 srgb} bind def /col18 {0.560 0.000 0.000 srgb} bind def /col19 {0.690 0.000 0.000 srgb} bind def /col20 {0.820 0.000 0.000 srgb} bind def /col21 {0.560 0.000 0.560 srgb} bind def /col22 {0.690 0.000 0.690 srgb} bind def /col23 {0.820 0.000 0.820 srgb} bind def /col24 {0.500 0.190 0.000 srgb} bind def /col25 {0.630 0.250 0.000 srgb} bind def /col26 {0.750 0.380 0.000 srgb} bind def /col27 {1.000 0.500 0.500 srgb} bind def /col28 {1.000 0.630 0.630 srgb} bind def /col29 {1.000 0.750 0.750 srgb} bind def /col30 {1.000 0.880 0.880 srgb} bind def /col31 {1.000 0.840 0.000 srgb} bind def /col32 {0.776 0.784 0.784 srgb} bind def /col33 {0.776 0.784 0.784 srgb} bind def end save -38.0 302.0 translate 1 -1 scale /cp {closepath} bind def /ef {eofill} bind def /gr {grestore} bind def /gs {gsave} bind def /sa {save} bind def /rs {restore} bind def /l {lineto} bind def /m {moveto} bind def /rm {rmoveto} bind def /n {newpath} bind def /s {stroke} bind def /sh {show} bind def /slc {setlinecap} bind def /slj {setlinejoin} bind def /slw {setlinewidth} bind def /srgb {setrgbcolor} bind def /rot {rotate} bind def /sc {scale} bind def /sd {setdash} bind def /ff {findfont} bind def /sf {setfont} bind def /scf {scalefont} bind def /sw {stringwidth} bind def /tr {translate} bind def /tnt {dup dup currentrgbcolor 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} bind def /shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul srgb} bind def /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 10 setmiterlimit n -1000 6026 m -1000 -1000 l 13927 -1000 l 13927 6026 l cp clip 0.06000 0.06000 sc % Polyline 7.500 slw n 8100 3425 m 12900 3425 l gs col33 s gr % Polyline n 1500 2650 m 6300 2650 l gs col32 s gr % Polyline n 1500 2250 m 6300 2250 l gs col32 s gr % Polyline n 1500 1825 m 6300 1825 l gs col32 s gr /Times-Roman ff 270.00 scf sf 7500 2175 m gs 1 -1 sc 90.0 rot (Write Bandwidth \(MB/s\)) dup sw pop 2 div neg 0 rm col0 sh gr /Times-Roman ff 270.00 scf sf 900 2175 m gs 1 -1 sc 90.0 rot (Write Bandwidth \(MB/s\)) dup sw pop 2 div neg 0 rm col0 sh gr % Polyline n 1500 1025 m 6300 1025 l gs col32 s gr % Polyline n 1500 1425 m 6300 1425 l gs col32 s gr % Polyline n 8100 1075 m 12900 1075 l gs col33 s gr % Polyline n 8100 1540 m 12900 1540 l gs col33 s gr % Polyline n 8100 2950 m 12900 2950 l gs col33 s gr % Polyline n 1500 3475 m 6300 3475 l gs col32 s gr % Polyline n 1500 3075 m 6300 3075 l gs col32 s gr % Polyline n 8100 2475 m 12900 2475 l gs col33 s gr % Polyline n 8100 2025 m 12900 2025 l gs col33 s gr % Polyline n 8400 825 m 10275 825 l 10275 1650 l 8400 1650 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 1800 825 m 3675 825 l 3675 1650 l 1800 1650 l cp gs col7 1.00 shd ef gr gs col-1 s gr % Polyline n 12915 1542 m 12840 1542 l gs col-1 s gr % Polyline n 12915 2014 m 12840 2014 l gs col-1 s gr % Polyline n 12915 598 m 12840 598 l gs col-1 s gr /Times-Roman ff 240.00 scf sf 8700 4200 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 9900 4200 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 11100 4200 m gs 1 -1 sc (32) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 12300 4200 m gs 1 -1 sc (64) dup sw pop 2 div neg 0 rm col-1 sh gr % Polyline n 12915 2486 m 12840 2486 l gs col-1 s gr % Polyline n 12915 2958 m 12840 2958 l gs col-1 s gr % Polyline n 12915 3430 m 12840 3430 l gs col-1 s gr % Polyline n 12915 3902 m 12840 3902 l gs col-1 s gr % Polyline n 12915 1070 m 12840 1070 l gs col-1 s gr /Times-Roman ff 240.00 scf sf 5700 4200 m gs 1 -1 sc (64) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 4500 4200 m gs 1 -1 sc (32) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 3300 4200 m gs 1 -1 sc (16) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 2100 4200 m gs 1 -1 sc (8) dup sw pop 2 div neg 0 rm col-1 sh gr % Polyline n 6315 3902 m 6240 3902 l gs col-1 s gr % Polyline n 6315 3489 m 6240 3489 l gs col-1 s gr % Polyline n 6315 3076 m 6240 3076 l gs col-1 s gr % Polyline n 6315 2663 m 6240 2663 l gs col-1 s gr % Polyline n 6315 2250 m 6240 2250 l gs col-1 s gr % Polyline n 6315 1837 m 6240 1837 l gs col-1 s gr % Polyline n 6315 1424 m 6240 1424 l gs col-1 s gr % Polyline n 6315 1011 m 6240 1011 l gs col-1 s gr % Polyline n 6315 598 m 6240 598 l gs col-1 s gr % Polyline n 1500 3900 m 6315 3902 l 6315 598 l 1500 598 l 1500 3902 l gs col-1 s gr % Polyline n 1483 3902 m 1558 3902 l gs col-1 s gr % Polyline n 1483 3489 m 1558 3489 l gs col-1 s gr % Polyline n 1483 3076 m 1558 3076 l gs col-1 s gr % Polyline n 1483 2663 m 1558 2663 l gs col-1 s gr % Polyline n 1483 2250 m 1558 2250 l gs col-1 s gr % Polyline n 1483 1837 m 1558 1837 l gs col-1 s gr % Polyline n 1483 1424 m 1558 1424 l gs col-1 s gr % Polyline n 1483 1011 m 1558 1011 l gs col-1 s gr % Polyline n 1483 598 m 1558 598 l gs col-1 s gr % Polyline n 1786 3902 m 1786 3533 l 2087 3533 l 2087 3902 l 1786 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 2918 3902 m 2918 3340 l 3220 3340 l 3220 3902 l 2918 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 4125 3902 m 4125 3156 l 4427 3156 l 4427 3902 l 4125 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 5332 3902 m 5332 1887 l 5634 1887 l 5634 3902 l 5332 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 2164 3902 m 2164 2861 l 2466 2861 l 2466 3902 l 2164 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 3296 3902 m 3296 2580 l 3598 2580 l 3598 3902 l 3296 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 4503 3902 m 4503 2126 l 4805 2126 l 4805 3902 l 4503 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 5711 3902 m 5711 1217 l 6012 1217 l 6012 3902 l 5711 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 9940 3902 m 9940 2569 l 10237 2569 l 10237 3902 l 9940 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 8550 1350 m 8700 1350 l 8700 1500 l 8550 1500 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 8550 975 m 8700 975 l 8700 1125 l 8550 1125 l cp gs col29 1.00 shd ef gr gs col-1 s gr /Times-Roman ff 270.00 scf sf 2640 4950 m gs 1 -1 sc (\(a\) Small Data Size) col0 sh gr /Times-Roman ff 270.00 scf sf 10425 4500 m gs 1 -1 sc (Number of Processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 225.00 scf sf 9465 1125 m gs 1 -1 sc (Parallel HDF5) dup sw pop 2 div neg 0 rm col-1 sh gr % Polyline n 8082 3902 m 8157 3902 l gs col-1 s gr % Polyline n 8082 3430 m 8157 3430 l gs col-1 s gr % Polyline n 8082 2958 m 8157 2958 l gs col-1 s gr % Polyline n 8082 2486 m 8157 2486 l gs col-1 s gr % Polyline n 8082 2014 m 8157 2014 l gs col-1 s gr % Polyline n 8082 1542 m 8157 1542 l gs col-1 s gr % Polyline n 8082 1070 m 8157 1070 l gs col-1 s gr % Polyline n 8082 598 m 8157 598 l gs col-1 s gr % Polyline n 8082 3900 m 12900 3900 l 12900 600 l 8082 600 l 8082 3902 l gs col-1 s gr % Polyline n 10760 3902 m 10760 2658 l 11058 2658 l 11058 3902 l 10760 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 11951 3902 m 11951 2474 l 12248 2474 l 12248 3902 l 11951 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 8400 3902 m 8400 3487 l 8697 3487 l 8697 3902 l 8400 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 9569 3902 m 9569 3173 l 9867 3173 l 9867 3902 l 9569 3902 l cp gs col29 1.00 shd ef gr gs col-1 s gr % Polyline n 8749 3902 m 8749 3130 l 9047 3130 l 9047 3902 l 8749 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr /Times-Roman ff 225.00 scf sf 2865 1125 m gs 1 -1 sc (Parallel HDF5) dup sw pop 2 div neg 0 rm col-1 sh gr % Polyline n 11130 3902 m 11130 1235 l 11428 1235 l 11428 3902 l 11130 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 12321 3902 m 12321 1164 l 12619 1164 l 12619 3902 l 12321 3902 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 1950 1350 m 2100 1350 l 2100 1500 l 1950 1500 l cp gs col9 1.00 shd ef gr gs col-1 s gr % Polyline n 1950 975 m 2100 975 l 2100 1125 l 1950 1125 l cp gs col29 1.00 shd ef gr gs col-1 s gr /Times-Roman ff 270.00 scf sf 3750 4500 m gs 1 -1 sc (Number of Processors) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 270.00 scf sf 9375 4950 m gs 1 -1 sc (\(b\) Large Data Size) col0 sh gr /Times-Roman ff 225.00 scf sf 9487 1500 m gs 1 -1 sc (Prallel netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 225.00 scf sf 2887 1500 m gs 1 -1 sc (Prallel netCDF) dup sw pop 2 div neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 3964 m gs 1 -1 sc (0) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 660 m gs 1 -1 sc (140) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 1132 m gs 1 -1 sc (120) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 1604 m gs 1 -1 sc (100) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 2076 m gs 1 -1 sc (80) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 1073 m gs 1 -1 sc (35) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 3551 m gs 1 -1 sc (5) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 2548 m gs 1 -1 sc (60) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 3020 m gs 1 -1 sc (40) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 3492 m gs 1 -1 sc (20) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 8008 3964 m gs 1 -1 sc (0) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 3138 m gs 1 -1 sc (10) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 1486 m gs 1 -1 sc (30) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 660 m gs 1 -1 sc (40) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 2725 m gs 1 -1 sc (15) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 2312 m gs 1 -1 sc (20) dup sw pop neg 0 rm col-1 sh gr /Times-Roman ff 240.00 scf sf 1409 1899 m gs 1 -1 sc (25) dup sw pop neg 0 rm col-1 sh gr $F2psEnd rs %%EndDocument @endspecial -14 1589 a Fe(Figure)30 b(7.)369 1489 y SDict begin H.S end 369 1489 a 369 1489 a SDict begin H.R end 369 1489 a 369 1489 a SDict begin [ /View [/XYZ H.V] /Dest (figure.7) cvn H.B /DEST pdfmark end 369 1489 a 100 x Fe(P)n(erf)n(ormance)h(of)f(FLASH)g(I/O)g(benc)o(hmark)h(using) e(parallel)i(HDF5)g(and)f(parallel)h(netCDF)-8 b(.)29 b(The)h(tw)n(o)-14 1689 y(e)o(xperiments)21 b(use)g(diff)o(erent)f (parameter)o(s)i(so)f(that)f(the)h(\002le)g(siz)q(es)g(are)h(diff)o (erent.)31 b(Also)21 b(the)g(\002le)g(siz)q(es)g(are)h(v)n(aries)-14 1788 y(with)k(the)h(n)o(umber)g(of)f(pr)n(ocessor)o(s.)46 b(The)27 b(I/O)g(amount)f(is)h(3MB)h(*)f(Number)g(of)g(Pr)n(ocessor)o (s)i(in)e(\(a\),)h(and)e(24MB)i(*)-14 1888 y(Number)22 b(of)h(Pr)n(ocessor)o(s)h(in)f(\(b\).)-114 2237 y Fg(slab)28 b(used)g(for)f(parallel)g(access,)j(which)d(mak)o(es)h(the)f(packing)g (of)-114 2337 y(the)19 b(hyperslabs)e(into)i(contiguous)d(b)n(uf)n (fers)i(tak)o(e)h(a)g(relati)n(v)o(ely)f(long)-114 2437 y(time.)-114 2557 y SDict begin H.S end -114 2557 a -114 2557 a SDict begin 12 H.A end -114 2557 a -114 2557 a SDict begin [ /View [/XYZ H.V] /Dest (section.6) cvn H.B /DEST pdfmark end -114 2557 a 119 x Fi(6)99 b(Conclusion)25 b(and)h(Futur)n(e)g(W)-7 b(ork)-14 2899 y Fg(In)29 b(this)h(w)o(ork,)h (we)f(e)o(xtend)f(the)g(serial)h(netCDF)g(interf)o(ace)f(to)-114 2999 y(f)o(acilitate)j(parallel)g(access,)j(and)c(we)h(pro)o(vide)e(an) i(implementa-)-114 3099 y(tion)23 b(for)g(a)h(subset)g(of)g(this)g(ne)n (w)f(parallel)g(netCDF)h(interf)o(ace.)35 b(By)-114 3198 y(b)n(uilding)24 b(on)i(top)f(of)h(MPI-IO,)f(we)h(gain)f(a)h(number)e (of)h(interf)o(ace)-114 3298 y(adv)n(antages)c(and)h(performance)e (optimizations)i(users)h(can)f(bene-)-114 3397 y(\002t)d(from)d(by)i (using)f(this)h(parallel)f(netCDF)i(package,)d(as)j(sho)n(wn)e(by)-114 3497 y(our)e(test)j(results.)23 b(So)17 b(f)o(ar)m(,)f(a)h(number)d(of) i(users)h(from)e(LBL,)h(ORNL,)-114 3597 y(and)j(Uni)n(v)o(ersity)f(of)h (Chicago)f(are)h(using)g(our)f(parallel)h(netCDF)h(li-)-114 3696 y(brary)-5 b(.)-14 3800 y(Future)36 b(w)o(ork)g(in)m(v)n(olv)o(es) f(de)n(v)o(eloping)f(a)k(production-quality)-114 3899 y(parallel)27 b(netCDF)i(API)f(\(for)f(C,)i(C++,)h(F)o(ortran,)e(and)f (other)g(pro-)-114 3999 y(gramming)22 b(languages\))g(and)i(making)e (it)j(freely)e(a)n(v)n(ailable)h(to)g(the)-114 4098 y(high-performance) 31 b(computing)j(community)-5 b(.)69 b(Moreo)o(v)o(er)m(,)37 b(we)-114 4198 y(need)23 b(to)i(de)n(v)o(elop)d(a)i(mechanism)f(for)g (matching)g(the)h(\002le)h(or)o(gani-)-114 4298 y(zation)c(to)i(access) f(patterns,)g(and)f(we)i(need)e(to)h(de)n(v)o(elop)e(cross-\002le)-114 4397 y(optimizations)f(for)g(addressing)g(common)g(data)h(access)g (patterns.)-114 4612 y Ff(Ackno)o(wledgements)-114 4742 y SDict begin H.S end -114 4742 a -114 4742 a SDict begin 12 H.A end -114 4742 a -114 4742 a SDict begin [ /View [/XYZ H.V] /Dest (section*.1) cvn H.B /DEST pdfmark end -114 4742 a -14 4827 a Fg(This)k(w)o(ork)f(is)i(sponsored)e(by)h (Scienti\002c)g(Data)g(Management)-114 4926 y(Center)34 b(of)g(DOE)g(SciD)m(A)m(C)h(ISICs)g(program)d(and)i(jointly)f(con-)-114 5026 y(ducted)e(at)h(Northwestern)e(Uni)n(v)o(ersity)g(and)i(Ar)o (gonne)d(National)-114 5126 y(Laboratory)-5 b(.)59 b(This)33 b(research)e(w)o(as)j(also)e(supported)f(in)i(part)f(by)-114 5225 y(NSF)19 b(cooperati)n(v)o(e)d(agreement)g(A)m(CI-9619020)f (through)h(comput-)-114 5325 y(ing)22 b(resources)f(pro)o(vided)f(by)i (the)g(National)f(P)o(artnership)g(for)h(Ad-)2042 2237 y(v)n(anced)j(Computational)f(Infrastructure)g(at)j(the)f(San)g(Die)o (go)g(Su-)2042 2337 y(percomputer)18 b(Center)-5 b(.)2142 2446 y(W)e(e)21 b(thank)d(W)-7 b(oo-Sun)19 b(Y)-8 b(ang)19 b(from)f(LBL)i(for)f(pro)o(viding)e(us)j(the)2042 2546 y(test)27 b(code)e(for)g(performance)e(e)n(v)n(aluation)h(and)i(Nagiza) f(F)-7 b(.)27 b(Sama-)2042 2645 y(to)o(v)n(a)17 b(and)g(Da)n(vid)g (Bauer)g(at)h(ORNL)g(for)f(using)g(our)g(library)f(and)h(for)2042 2745 y(gi)n(ving)i(us)i(feedback)d(and)i(v)n(aluable)f(suggestions.) 2042 2996 y Fi(Refer)n(ences)2042 3123 y SDict begin H.S end 2042 3123 a 2042 3123 a SDict begin 12 H.A end 2042 3123 a 2042 3123 a SDict begin [ /View [/XYZ H.V] /Dest (section*.2) cvn H.B /DEST pdfmark end 2042 3123 a 2042 3123 a SDict begin H.S end 2042 3123 a 2042 3123 a SDict begin 11 H.A end 2042 3123 a 2042 3123 a SDict begin [ /View [/XYZ H.V] /Dest (cite.FORT00) cvn H.B /DEST pdfmark end 2042 3123 a 2093 3183 a Fc([1])42 b(B.)22 b(Fryx)o(ell,)i(K.)e (Olson,)i(P)-8 b(.)22 b(Rick)o(er)m(,)h(F)-6 b(.)22 b(X.)h(T)m(immes,)h (M.)e(Zingale,)2222 3275 y(D.)14 b(Q.)g(Lamb,)h(P)-8 b(.)13 b(MacNeice,)j(R.)e(Rosner)m(,)h(and)h(H.)e(T)m(ufo.)f (\223FLASH:)2222 3366 y(An)19 b(adapti)n(v)o(e)g(mesh)g(hydrodynamics)i (code)e(for)g(modelling)g(astro-)2222 3457 y(physical)e(thermonuclear)i (\003ashes,)-5 b(\224)17 b Fb(Astr)m(ophysical)g(J)n(ournal)h(Sup-)2222 3548 y(pliment)p Fc(,)h(2000,)g(pp.)g(131-273.)2042 3617 y SDict begin H.S end 2042 3617 a 2042 3617 a SDict begin 11 H.A end 2042 3617 a 2042 3617 a SDict begin [ /View [/XYZ H.V] /Dest (cite.GLDS96) cvn H.B /DEST pdfmark end 2042 3617 a 2093 3662 a Fc([2])42 b(W)-7 b(.)30 b(Gropp,)k(E.)c(Lusk,)k(N.)d (Doss,)i(and)f(A.)e(Skjellum.)g(\223)-6 b(A)31 b(high-)2222 3753 y(performance,)19 b(portable)g(implementation)f(of)g(the)g(MPI)g (Message-)2222 3845 y(P)o(assing)g(Interf)o(ace)g(standard,)-5 b(\224)19 b Fb(P)-6 b(ar)o(allel)18 b(Computing)p Fc(,)g(22\(6\):789-) 2222 3936 y(828,)i(1996.)2042 3998 y SDict begin H.S end 2042 3998 a 2042 3998 a SDict begin 11 H.A end 2042 3998 a 2042 3998 a SDict begin [ /View [/XYZ H.V] /Dest (cite.GrLT99) cvn H.B /DEST pdfmark end 2042 3998 a 2093 4049 a Fc([3])42 b(W)-7 b(.)17 b(Gropp,)i(E.)e(Lusk,)h(and)g(R.)g(Thakur)l (.)g Fb(Using)g(MPI-2:)k(Advanced)2222 4141 y(F)-6 b(eatur)m(es)17 b(of)e(the)h(Messa)o(g)o(e)h(P)-6 b(assing)16 b(Interface)p Fc(,)g(MIT)g(Press,)f(Cam-)2222 4232 y(bridge,)k(MA,)g(1999.)2042 4300 y SDict begin H.S end 2042 4300 a 2042 4300 a SDict begin 11 H.A end 2042 4300 a 2042 4300 a SDict begin [ /View [/XYZ H.V] /Dest (cite.HDF4) cvn H.B /DEST pdfmark end 2042 4300 a 2093 4345 a Fc([4])42 b(HDF4)15 b(Home)g(P)o(age.)f(The)h (National)g(Center)g(for)g(Supercomputing)2222 4437 y(Applications.)k Fb(http://)g(hdf)o(.ncsa.uiuc.edu/hdf4.html)p Fc(.)2042 4505 y SDict begin H.S end 2042 4505 a 2042 4505 a SDict begin 11 H.A end 2042 4505 a 2042 4505 a SDict begin [ /View [/XYZ H.V] /Dest (cite.HDF5) cvn H.B /DEST pdfmark end 2042 4505 a 2093 4550 a Fc([5])42 b(HDF5)15 b(Home)g(P)o(age.)f(The)h (National)g(Center)g(for)g(Supercomputing)2222 4642 y(Applications.)k Fb(http://)g(hdf)o(.ncsa.uiuc.edu/HDF5/)p Fc(.)2042 4710 y SDict begin H.S end 2042 4710 a 2042 4710 a SDict begin 11 H.A end 2042 4710 a 2042 4710 a SDict begin [ /View [/XYZ H.V] /Dest (cite.LLCT02) cvn H.B /DEST pdfmark end 2042 4710 a 2093 4755 a Fc([6])42 b(J.)27 b(Li,)i(W)-7 b(.)27 b(Liao,)i(A.)e(Choudhary)-5 b(,)31 b(and)e(V)-10 b(.)27 b(T)-6 b(aylor)l(.)27 b(\223I/O)h(Analy-)2222 4846 y(sis)19 b(and)i(Optimization)e(for)h(an)g(AMR)g(Cosmology)h(Application,)-5 b(\224)2222 4938 y(in)23 b Fb(Pr)m(oceedings)i(of)e(IEEE)f(Cluster)i (2002)p Fc(,)h(Chicago,)g(September)2222 5029 y(2002.)2042 5082 y SDict begin H.S end 2042 5082 a 2042 5082 a SDict begin 11 H.A end 2042 5082 a 2042 5082 a SDict begin [ /View [/XYZ H.V] /Dest (cite.Mess97) cvn H.B /DEST pdfmark end 2042 5082 a 2093 5142 a Fc([7])42 b(Message)66 b(P)o(assing)e(Interf) o(ace)h(F)o(orum.)f(\223MPI-2:)114 b(Exten-)2222 5234 y(sions)60 b(to)f(the)g(Message-P)o(assing)h(Interf)o(ace\224,)69 b(July)60 b(1997.)2222 5340 y SDict begin H.S end 2222 5340 a -15 x Fb(http://www)-6 b(.mpi-forum.or)m(g/docs/docs.html)3470 5274 y SDict begin H.R end 3470 5274 a 3470 5325 a SDict begin [ /H /I /Border [0 0 12] /Color [0 1 1] /Action << /Subtype /URI /URI (http://www.mpi-forum.org/docs/docs.html) >> /Subtype /Link H.B /ANN pdfmark end 3470 5325 a Fc(.)1928 5574 y Fg(9)p eop %%Page: 10 10 10 9 bop 0 0 a SDict begin /product where{pop product(Distiller)search{pop pop pop version(.)search{exch pop exch pop(3011)eq{gsave newpath 0 0 moveto closepath clip/Courier findfont 10 scalefont setfont 72 72 moveto(.)show grestore}if}{pop}ifelse}{pop}ifelse}if end 0 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin H.R end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (page.10) cvn H.B /DEST pdfmark end -114 0 a -114 0 a SDict begin H.S end -114 0 a -114 0 a SDict begin 11 H.A end -114 0 a -114 0 a SDict begin [ /View [/XYZ H.V] /Dest (cite.RDED97) cvn H.B /DEST pdfmark end -114 0 a -63 83 a Fc([8])42 b(R.)23 b(Re)n(w)-5 b(,)24 b(G.)f(Da)o(vis,)g(S.)g(Emmerson,)h(and)h(H.)d(Da)o(vies,)i (\223NetCDF)66 174 y(User')l(s)j(Guide)g(for)g(C,)-5 b(\224)27 b(Unidata)g(Program)h(Center)m(,)h(June)f(1997.)66 281 y SDict begin H.S end 66 281 a -15 x Fb(http://www)-6 b(.unidata.ucar)e(.edu/pac)o(ka)o (g)o(es/netcdf/guidec/)1653 215 y SDict begin H.R end 1653 215 a 1653 266 a SDict begin [ /H /I /Border [0 0 12] /Color [0 1 1] /Action << /Subtype /URI /URI (http://www.unidata.ucar.edu/packages/netcdf/guidec/) >> /Subtype /Link H.B /ANN pdfmark end 1653 266 a Fc(.)-114 314 y SDict begin H.S end -114 314 a -114 314 a SDict begin 11 H.A end -114 314 a -114 314 a SDict begin [ /View [/XYZ H.V] /Dest (cite.ReDa90) cvn H.B /DEST pdfmark end -114 314 a -63 360 a Fc([9])42 b(R.)36 b(Re)n(w)i(and)f(G.)g(Da)o(vis,)k(\223The)d(Unidata)f(netCDF:)g(Softw)o (are)66 452 y(for)22 b(Scienti\002c)g(Data)g(Access,)-5 b(\224)24 b Fb(Sixth)e(International)i(Confer)m(ence)66 543 y(on)e(Inter)o(active)g(Information)g(and)h(Pr)m(ocessing)f (Systems)g(for)f(Me-)66 634 y(teor)m(olo)o(gy)l(,)40 b(Oceano)o(gr)o(aphy)e(and)f(Hydr)m(olo)o(gy)p Fc(,)j(Anaheim,)g(CA,)66 726 y(February)19 b(1990.)-114 775 y SDict begin H.S end -114 775 a -114 775 a SDict begin 11 H.A end -114 775 a -114 775 a SDict begin [ /View [/XYZ H.V] /Dest (cite.RNCZ01) cvn H.B /DEST pdfmark end -114 775 a -100 820 a Fc([10])42 b(R.)30 b(Ross,)j(D.)e(Nurmi,)i(A.)d(Cheng,)35 b(and)c(M.)g(Zingale,)i (\223)-6 b(A)30 b(Case)66 912 y(Study)16 b(in)g(Application)g(I/O)f(on) i(Linux)f(Clusters\224,)g(in)g Fb(Pr)m(oceedings)66 1003 y(of)j(SC2001)p Fc(,)h(Den)m(v)o(er)m(,)f(No)o(v)o(ember)h(2001.)-114 1052 y SDict begin H.S end -114 1052 a -114 1052 a SDict begin 11 H.A end -114 1052 a -114 1052 a SDict begin [ /View [/XYZ H.V] /Dest (cite.RoBC93) cvn H.B /DEST pdfmark end -114 1052 a -100 1098 a Fc([11])42 b(J.M.)15 b(Rosario,)h(R.)f(Borda) o(wekar)m(,)i(and)f(A.)f(Choudhary)-5 b(.)17 b(\223Impro)o(v)o(ed)66 1189 y(P)o(arallel)31 b(I/O)h(via)g(a)g(T)-6 b(w)o(o-Phase)33 b(Run-time)f(Access)h(Strate)o(gy)-5 b(,)g(\224)66 1280 y Fb(IPPS)18 b('93)h(P)-6 b(ar)o(allel)19 b(I/O)f(W)-7 b(orkshop)p Fc(,)21 b(February)e(9,)g(1993.)-114 1330 y SDict begin H.S end -114 1330 a -114 1330 a SDict begin 11 H.A end -114 1330 a -114 1330 a SDict begin [ /View [/XYZ H.V] /Dest (cite.TBCP94) cvn H.B /DEST pdfmark end -114 1330 a -100 1375 a Fc([12])42 b(R.)26 b(Thakur)m(,)j(R.)d(Borda)o (wekar)m(,)j(A.)d(Choudhary)-5 b(,)30 b(R.)c(Ponnusamy)-5 b(,)66 1466 y(and)19 b(T)-6 b(.)18 b(Singh.)g(\223P)-7 b(ASSION)16 b(Runtime)j(Library)f(for)g(P)o(arallel)f(I/O\224,)66 1557 y Fb(Scalable)j(P)-6 b(ar)o(allel)18 b(Libr)o(aries)h(Confer)m (ence)p Fc(,)h(Oct.)e(1994.)-114 1606 y SDict begin H.S end -114 1606 a -114 1606 a SDict begin 11 H.A end -114 1606 a -114 1606 a SDict begin [ /View [/XYZ H.V] /Dest (cite.ThCh96) cvn H.B /DEST pdfmark end -114 1606 a -100 1652 a Fc([13])42 b(R.)35 b(Thakur)h(and)g(A.)f(Choudhary)-5 b(.)38 b(\223)-6 b(An)36 b(Extended)g(T)-6 b(w)o(o-Phase)66 1743 y(Method)17 b(for)f(Accessing)h(Sections)f(of)g(Out-of-Core)g (Arrays,)-5 b(\224)17 b Fb(Sci-)66 1835 y(enti\002c)i(Pr)m(o)o(gr)o (amming)p Fc(,)g(5\(4\):301-317,)i(W)m(inter)d(1996.)-114 1883 y SDict begin H.S end -114 1883 a -114 1883 a SDict begin 11 H.A end -114 1883 a -114 1883 a SDict begin [ /View [/XYZ H.V] /Dest (cite.ThGL96) cvn H.B /DEST pdfmark end -114 1883 a -100 1929 a Fc([14])42 b(R.)30 b(Thakur)m(,)k(W)-7 b(.)30 b(Gropp,)k(and)d(E.)f(Lusk.)h(\223)-6 b(An)31 b(Abstract-De)n(vice)66 2021 y(interf)o(ace)52 b(for)f(Implementing)i (Portable)e(P)o(arallel-I/O)f(Inter)o(-)66 2112 y(f)o (aces\224\(ADIO\),)21 b(in)g Fb(Pr)m(oceedings)h(of)f(the)h(6th)f (Symposium)i(on)e(the)66 2203 y(F)l(r)m(ontier)o(s)c(of)g(Massively)h (P)-6 b(ar)o(allel)17 b(Computation)p Fc(,)h(October)g(1996,)66 2295 y(pp.)h(180-187.)-114 2344 y SDict begin H.S end -114 2344 a -114 2344 a SDict begin 11 H.A end -114 2344 a -114 2344 a SDict begin [ /View [/XYZ H.V] /Dest (cite.ThGL99) cvn H.B /DEST pdfmark end -114 2344 a -100 2389 a Fc([15])42 b(R.)21 b(Thakur)m(,)h(W)-7 b(.)21 b(Gropp,)i(and)f(E.)f(Lusk.)g (\223Data)h(Sie)n(ving)g(and)g(Col-)66 2481 y(lecti)n(v)o(e)c(I/O)f(in) g(R)m(OMIO,)-5 b(\224)18 b(in)f Fb(Pr)m(oceeding)i(of)f(the)f(7th)h (Symposium)66 2572 y(on)j(the)g(F)l(r)m(ontier)o(s)g(of)g(Massively)h (P)-6 b(ar)o(allel)20 b(Computation)p Fc(,)j(Febru-)66 2663 y(ary)c(1999,)h(pp.)f(182-189.)-114 2713 y SDict begin H.S end -114 2713 a -114 2713 a SDict begin 11 H.A end -114 2713 a -114 2713 a SDict begin [ /View [/XYZ H.V] /Dest (cite.ThGL99b) cvn H.B /DEST pdfmark end -114 2713 a -100 2758 a Fc([16])42 b(R.)17 b(Thakur)m(,)h(W)-7 b(.)17 b(Gropp,)i(and)f(E.)f(Lusk.)h(\223On)g(implementing)h(MPI-)66 2849 y(IO)j(portably)h(and)g(with)e(high)i(performance,)-5 b(\224)24 b(in)e Fb(Pr)m(oceedings)i(of)66 2941 y(the)32 b(Sixth)g(W)-7 b(orkshop)34 b(on)f(Input/Output)g(in)f(P)-6 b(ar)o(allel)31 b(and)i(Dis-)66 3032 y(trib)o(uted)19 b(Systems)p Fc(,)g(May)g(1999,)h(pp.)f(23-32.)-114 3081 y SDict begin H.S end -114 3081 a -114 3081 a SDict begin 11 H.A end -114 3081 a -114 3081 a SDict begin [ /View [/XYZ H.V] /Dest (cite.TRLG02) cvn H.B /DEST pdfmark end -114 3081 a -100 3127 a Fc([17])42 b(R.)23 b(Thakur)m(,)i(R.)e(Ross,)h(E.)f (Lusk,)h(and)h(W)-7 b(.)22 b(Gropp,)j(\223Users)f(Guide)66 3218 y(for)c(R)m(OMIO:)g(A)g(High-Performance,)i(Portable)e(MPI-IO)g (Imple-)66 3309 y(mentation,)-5 b(\224)21 b(T)-5 b(echnical)21 b(Memorandum)i(No.)e(234,)h(Mathematics)66 3401 y(and)f(Computer)g (Science)g(Di)n(vision,)f(Ar)o(gonne)i(National)e(Labora-)66 3492 y(tory)-5 b(,)19 b(Re)n(vised)g(January)h(2002.)-114 3541 y SDict begin H.S end -114 3541 a -114 3541 a SDict begin 11 H.A end -114 3541 a -114 3541 a SDict begin [ /View [/XYZ H.V] /Dest (cite.FLASHIO) cvn H.B /DEST pdfmark end -114 3541 a -100 3587 a Fc([18])42 b(M.)27 b(Zingale.)g(FLASH)f(I/O)h (benchmark.)1222 3602 y SDict begin H.S end 1222 3602 a -15 x Fb(http://\003ash.uc)o(hica)o (go)1836 3535 y SDict begin H.R end 1836 3535 a 1836 3587 a SDict begin [ /H /I /Border [0 0 12] /Color [0 1 1] /Action << /Subtype /URI /URI (http://flash.uchicago) >> /Subtype /Link H.B /ANN pdfmark end 1836 3587 a Fb(.)66 3678 y(edu/)p Fa(\230)p Fb(zingale/\003ash)p 619 3678 23 4 v 29 w(benc)o(hmark)p 977 3678 V 29 w(io/)p Fc(.)-114 3727 y SDict begin H.S end -114 3727 a -114 3727 a SDict begin 11 H.A end -114 3727 a -114 3727 a SDict begin [ /View [/XYZ H.V] /Dest (cite.WINU) cvn H.B /DEST pdfmark end -114 3727 a -100 3773 a Fc([19])42 b(Where)27 b(is)g(NetCDF)f(Used?) 49 b(Unidata)27 b(Program)g(Center)l(.)g Fb(http://)66 3864 y(www)-6 b(.unidata.ucar)e(.edu/pac)o(ka)o(g)o(es/netcdf/usa)o(g)o (e)o(.html)p Fc(.)1907 5574 y Fg(10)p eop %%Trailer end end userdict /end-hook known{end-hook}if %%EOF