{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"7"},"type":{"value":"flat"},"separator":{"value":"pipe"}},"macroMetadata":{"macroId":{"value":"8722172309c4fd5c585d35bd7c9fc89b"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"7165b0bc-57eb-4927-a001-bded55363a39"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"Overview","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Rmpi","type":"text","marks":[{"type":"link","attrs":{"href":"https://cran.r-project.org/web/packages/Rmpi/index.html"}}]},{"text":": An interface (wrapper) to MPI. It also provides interactive R manager and worker environment.","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Using","type":"text"}]},{"type":"paragraph","content":[{"text":"This usage relates to using this library on Beartooth.","type":"text"}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"We currently only have version 0.7.1 available.","type":"text"}]},{"type":"paragraph","content":[{"text":"Note","type":"text","marks":[{"type":"strong"}]},{"text":": Loading this module will also load in ","type":"text"},{"text":"r/4.2.2","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Meaning you have an environment with ","type":"text"},{"text":"r/4.2.2","type":"text","marks":[{"type":"code"}]},{"text":" and the ","type":"text"},{"text":"Rmpi","type":"text","marks":[{"type":"code"}]},{"text":" library.","type":"text"}]},{"type":"paragraph","content":[{"text":"You do NOT need to load ","type":"text"},{"text":"r/4.2.2","type":"text","marks":[{"type":"code"}]},{"text":" as a module separately.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Multicore","type":"text"}]},{"type":"paragraph","content":[{"text":"This R library is designed to run across multiple nodes, and multiple tasks on a node.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"ONLY using the Rmpi library","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"If you are only using the ","type":"text"},{"text":"Rmpi","type":"text","marks":[{"type":"code"}]},{"text":" library, and no other related parallel libraries such as ","type":"text"},{"text":"snow","type":"text","marks":[{"type":"code"}]},{"text":", then to allow the use of Rmpi on the cluster you first need to copy the following file into your home folder. (If you already have a ","type":"text"},{"text":".Rprofile","type":"text","marks":[{"type":"code"}]},{"text":" file in your home, you'll need to update it.)","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[]$ cp /apps/u/spack/gcc/12.2.0/r-rmpi/0.7-1-3eiutsq/rlib/R/library/Rmpi/Rprofile ~/.Rprofile","type":"text"}]},{"type":"paragraph","content":[{"text":"To then use the module, you’ll need to load the following modules:","type":"text"}]},{"type":"codeBlock","content":[{"text":"module load gcc/12.2.0 openmpi/4.1.4 r-rmpi/0.7-1-ompi","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Example","type":"text"}]},{"type":"paragraph","content":[{"text":"This example was based on an example ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://help.rc.ufl.edu/doc/R_MPI_Example"}}]},{"text":":","type":"text"}]},{"type":"expand","attrs":{"title":"rmpi_test.R"},"content":[{"type":"codeBlock","content":[{"text":"# Load the R MPI package if it is not already loaded.\nif (!is.loaded(\"mpi_initialize\")) {\n library(\"Rmpi\")\n }\n\nns <- mpi.universe.size() - 1\n\nmpi.spawn.Rslaves(nslaves=ns)\n\n# In case R exits unexpectedly, have it automatically clean up\n# resources taken up by Rmpi (slaves, memory, etc...)\n.Last <- function(){\n if (is.loaded(\"mpi_initialize\")){\n if (mpi.comm.size(1) > 0){\n print(\"Please use mpi.close.Rslaves() to close slaves.\")\n mpi.close.Rslaves()\n }\n print(\"Please use mpi.quit() to quit R\")\n .Call(\"mpi_finalize\")\n }\n}\n\n# Tell all slaves to return a message identifying themselves\nmpi.bcast.cmd( id <- mpi.comm.rank() )\nmpi.bcast.cmd( ns <- mpi.comm.size() )\nmpi.bcast.cmd( host <- mpi.get.processor.name() )\nmpi.remote.exec(paste(\"I am\",mpi.comm.rank(),\"of\",mpi.comm.size()))\n\n# Test computations\nx <- 5\nx <- mpi.remote.exec(rnorm, x)\nlength(x)\nx\n\n# Tell all slaves to close down, and exit the program\nmpi.close.Rslaves(dellog = FALSE)\nmpi.quit()","type":"text"}]}]},{"type":"expand","attrs":{"title":"run.sh"},"content":[{"type":"codeBlock","content":[{"text":"#!/bin/bash\n#SBATCH --job-name=rmpi-test\n#SBATCH --nodes=2\n#SBATCH --ntasks-per-node=4\n#SBATCH --time=10:00\n#SBATCH --mail-type=ALL\n#SBATCH --mail-user=\n#SBATCH --account=\n\nmodule load gcc/12.2.0 openmpi/4.1.4 r-rmpi/0.7-1-ompi\n\nsrun Rscript rmpi_test.R","type":"text"}]}]},{"type":"expand","attrs":{"title":"Example Run and Output"},"content":[{"type":"codeBlock","content":[{"text":"[]$ sbatch run_01.sh\nSubmitted batch job 13116893\n\n[]$ cat slurm-13116893.out\nmaster (rank 0, comm 1) of size 8 is running on: ttest01\nslave1 (rank 1, comm 1) of size 8 is running on: ttest01\nslave2 (rank 2, comm 1) of size 8 is running on: ttest01\nslave3 (rank 3, comm 1) of size 8 is running on: ttest01\nslave4 (rank 4, comm 1) of size 8 is running on: ttest02\nslave5 (rank 5, comm 1) of size 8 is running on: ttest02\nslave6 (rank 6, comm 1) of size 8 is running on: ttest02\nslave7 (rank 7, comm 1) of size 8 is running on: ttest02\nError in mpi.spawn.Rslaves(nslaves = ns) :\n It seems there are some slaves running on comm 1\n$slave1\n[1] \"I am 1 of 8\"\n\n$slave2\n[1] \"I am 2 of 8\"\n\n$slave3\n[1] \"I am 3 of 8\"\n\n$slave4\n[1] \"I am 4 of 8\"\n\n$slave5\n[1] \"I am 5 of 8\"\n\n$slave6\n[1] \"I am 6 of 8\"\n\n$slave7\n[1] \"I am 7 of 8\"\n\n[1] 7\n X1 X2 X3 X4 X5 X6\n1 0.25231568 -0.70670787 -0.8623333 -1.1538241 -1.3747273 -0.9696954\n2 -0.91498764 1.09819580 0.5737269 -0.6856323 0.8941616 -1.7339326\n3 1.51865169 1.63120359 -0.9954300 0.2413086 0.2627482 1.6690493\n4 1.09594877 0.08905511 -0.1490578 1.2190246 -0.1724257 -1.3822756\n5 0.09966169 -1.92527468 0.9805431 1.8346315 0.2773092 -0.7084154\n X7\n1 -0.26325262\n2 -1.20082024\n3 -0.04534522\n4 -0.14685414\n5 0.34071411\n[1] 1","type":"text"}]}]},{"type":"expand","attrs":{"title":"Error if no .Rprofile"},"content":[{"type":"codeBlock","content":[{"text":"# If you do not copy the .Rprofile file into you home, you'll see an error of the form:\nError in mpi.comm.spawn(slave = system.file(\"Rslaves.sh\", package = \"Rmpi\"), :\n MPI_ERR_SPAWN: could not spawn processes\nCalls: mpi.spawn.Rslaves -> mpi.comm.spawn\nExecution halted","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Using the Rmpi library with Snow","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"To start:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"This assumes you have installed the snow library yourself into your home.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"You know the location for this library: For example: ","type":"text"},{"text":"~/R/x86_64-pc-linux-gnu-library/4.2/snow/","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"You have NOT copied the ","type":"text"},{"text":".Rprofile","type":"text","marks":[{"type":"code"}]},{"text":" file into your home folder. If you have, then running this example will appear that the test has stalled and will eventually just time out.","type":"text"}]}]}]}]},{"type":"expand","attrs":{"title":"snow_test.R"},"content":[{"type":"codeBlock","content":[{"text":"library(Rmpi)\nlibrary(snow)\n\nsimu <- function(rep_worker, n_used) {\n theta_simu <- c()\n for (i in 1 : rep_worker) {\n theta_simu[i] <- mean(rnorm(n_used))\n }\n theta_simu\n}\n\nnum_of_processes <- mpi.universe.size()\nsprintf(\"Num of Processes: %d\", num_of_processes)\n\ncluster <- makeCluster(num_of_processes - 1, type = \"MPI\")\n\nn_used <- 1e4\nrep_worker_list <- rep(1, 100)\n\ntheta <- clusterApply(cluster, rep_worker_list, simu, n_used)\n\ntheta_cbind <- do.call(cbind, theta)\nwrite.csv(theta_cbind, file=\"values.csv\")\n\nstopCluster(cluster)","type":"text"}]}]},{"type":"expand","attrs":{"title":"run.sh"},"content":[{"type":"codeBlock","content":[{"text":"#!/bin/bash\n#SBATCH --job-name=rmpi_snow_test\n#SBATCH --nodes=2\n#SBATCH --ntasks-per-node=16\n#SBATCH --cpus-per-task=1\n#SBATCH --time=2:00\n#SBATCH --mail-type=ALL\n#SBATCH --mail-user=\n#SBATCH --account=\n\necho \"SLURM_JOB_ID:\" $SLURM_JOB_ID\n\n# Modules to Load\nmodule load gcc/12.2.0 openmpi/4.1.4 r-rmpi/0.7-1-ompi\n\n# https://stat.ethz.ch/pipermail/r-sig-hpc/2019-November/002105.html\n# This sets the PATH environment variable to where to find the RMPISNOW command.\nexport PATH=$PATH:~/R/x86_64-pc-linux-gnu-library/4.2/snow/\n\nmpirun RMPISNOW CMD BATCH --no-restore --no-save --quiet snow_test.R snow_test_$SLURM_JOB_ID.log","type":"text"}]}]},{"type":"expand","attrs":{"title":"Example Run and Output"},"content":[{"type":"codeBlock","content":[{"text":"[] ls\nrun.sh snow_test.R\n\n[]$ sbatch run.sh\nSubmitted batch job 13433495\n# Job should take less that 10 seconds to complete.\n\n[] ls\nrun.sh slurm-13433495.out snow_test_13433495.log values.csv snow_test.R\n\n[] cat slurm-13433495.out\nSLURM_JOB_ID: 13433495\n\n[]$ cat snow_test_13433495.log\nLoading required package: utils\n> library(Rmpi)\n> library(snow)\n>\n> simu <- function(rep_worker, n_used) {\n+ theta_simu <- c()\n+ for (i in 1 : rep_worker) {\n+ theta_simu[i] <- mean(rnorm(n_used))\n+ }\n+ theta_simu\n+ }\n>\n> num_of_processes <- mpi.universe.size()\n> sprintf(\"Num of Processes: %d\", num_of_processes)\n[1] \"Num of Processes: 32\"\n>\n> cluster <- makeCluster(num_of_processes - 1, type = \"MPI\")\n>\n> n_used <- 1e4\n> rep_worker_list <- rep(1, 100)\n>\n> theta <- clusterApply(cluster, rep_worker_list, simu, n_used)\n>\n> theta_cbind <- do.call(cbind, theta)\n> write.csv(theta_cbind, file=\"values.csv\")\n>\n> stopCluster(cluster)\n\n# Output will be of the form:\n[]$ cat values.csv\n\"\",\"V1\",\"V2\",\"V3\",\"V4\",\"V5\",\"V6\",\"V7\",\"V8\",\"V9\",\"V10\",\"V11\",\"V12\",\"V13\",\"V14\",\"V15\",\"V16\",\"V17\",\"V18\",\"V19\",\"V20\",\"V21\",\"V22\",\"V23\",\"V24\",\"V25\",\"V26\",\"V27\",\"V28\",\"V29\",\"V30\",\"V31\",\"V32\",\"V33\",\"V34\",\"V35\",\"V36\",\"V37\",\"V38\",\"V39\",\"V40\",\"V41\",\"V42\",\"V43\",\"V44\",\"V45\",\"V46\",\"V47\",\"V48\",\"V49\",\"V50\",\"V51\",\"V52\",\"V53\",\"V54\",\"V55\",\"V56\",\"V57\",\"V58\",\"V59\",\"V60\",\"V61\",\"V62\",\"V63\",\"V64\",\"V65\",\"V66\",\"V67\",\"V68\",\"V69\",\"V70\",\"V71\",\"V72\",\"V73\",\"V74\",\"V75\",\"V76\",\"V77\",\"V78\",\"V79\",\"V80\",\"V81\",\"V82\",\"V83\",\"V84\",\"V85\",\"V86\",\"V87\",\"V88\",\"V89\",\"V90\",\"V91\",\"V92\",\"V93\",\"V94\",\"V95\",\"V96\",\"V97\",\"V98\",\"V99\",\"V100\"\n\"1\",0.0110153488801525,0.00906305581302429,0.00108685858240707,0.00867668055186904,0.00578274342442965,-0.00530732478112944,0.0120954477596871,-0.00359434785869044,0.00835107071072111,2.64364921843532e-05,0.0152691100103968,-0.0135202565458591,0.00241255871463997,0.00137419862397849,0.00115252792794432,-0.0175922825490621,0.00110849307208272,-0.00937694151181359,-0.0131149112354201,0.00596388487426565,0.0219874640222224,-0.00547747285343229,0.00416837900267555,0.0139932426057719,-0.0234641772417162,0.00433451519201003,-0.00525816814860096,0.00414031282343361,0.00130366800166443,0.000413263824619443,0.0104087338213028,0.00149700313038921,0.011502836202072,-0.00715751527844509,0.0119589163613294,0.0220381656609346,-0.016000771903997,-0.00183947331801148,-0.00284276679070134,-0.00694346146022534,-0.0258218986912262,0.00473135994639517,0.00831409717862001,0.0182355174080247,-0.00931334883761317,0.00529566801009098,-0.00302557027855197,0.00346677904211363,0.00571337545701443,-0.00586232060572412,-0.00256997376593396,0.0165929336422106,0.00796493065507422,-0.00438474136670677,0.0062288833102191,-0.0175721248899911,0.00165933692067554,-0.00237930737409138,0.00121451126970138,0.00623046211970692,0.00559793460867063,-0.00412640828783677,-0.00764407338711362,-0.000460569630436792,-0.0107502392297747,-0.00421225031438457,-0.000926513045440252,0.00334739878419211,-0.00452111805160875,-0.0046740544706875,0.0155997050952078,-0.0234192042710321,0.00324579902597707,-0.0151148830758793,-0.000523464705140069,-0.00175640010460385,-0.010243166679217,-0.00668035373700306,-0.0119873621894053,-0.0141762507674786,-0.00783107010368886,0.0115902891884065,-0.00762658494377125,-0.0223384107212392,-0.00379425267947311,0.0138895890210734,0.00392029947365504,0.00248380077423007,-0.0064247327395136,0.00434147149528924,-0.00572841369840578,0.00966805999144852,-0.0122907653345613,0.00596172188548434,-0.0122757100311107,-0.000426327204500513,0.00108879897276763,0.00975469886227781,0.00675195747959386,-0.00288208828533988","type":"text"}]}]},{"type":"paragraph","content":[{"text":"This example is based on a script and discussion ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://stackoverflow.com/questions/41007564/stopcluster-in-r-snow-freeze"}}]},{"text":".","type":"text"}]},{"type":"paragraph"}],"version":1}

Browser not supported