{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Goal","type":"text","marks":[{"type":"strong"}]},{"text":": Demonstrate creating a module file to ","type":"text"},{"text":"load","type":"text","marks":[{"type":"em"}]},{"text":" a Conda environment.","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"1"},"outline":{"value":"false"},"style":{"value":"none"},"type":{"value":"list"},"printable":{"value":"true"}},"macroMetadata":{"macroId":{"value":"58f8685a-1da7-4b7c-a3e9-efa43779b1cd"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"904962f2-4f3e-4cf7-bcf9-64a1b8aa2799"}},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Remember what a module is","type":"text"}]},{"type":"paragraph","content":[{"text":"We use the LMOD module system to setup our environment by loading modules of compilers, languages, libraries and applications…","type":"text"}]},{"type":"paragraph","content":[{"text":"Although typically the module are setup by ARCC, you can create your own module files for software and environments that you have installed (within home/project). You can create your own module files that expose and load these.","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Module File Basic Template","type":"text"}]},{"type":"paragraph","content":[{"text":"An Introduction to Writing Modulefiles","type":"text","marks":[{"type":"link","attrs":{"href":"https://lmod.readthedocs.io/en/latest/015_writing_modules.html"}}]},{"text":": Module files are scripted using Lua and define core environment elements required to setup and use ","type":"text"},{"text":"something","type":"text","marks":[{"type":"em"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"Example","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ cd /project//software\n[] mkdir -p modules/tensorflow\n[] cd modules/tensorflow\n[] vim 2.16.lua","type":"text"}]},{"type":"codeBlock","content":[{"text":"# 2.16.lua\nwhatis(\" Name: TensorFlow \")\nwhatis(\" Version : 2.16 \")\nwhatis(\" Short Description: An end-to-end platform for machine learning.\")\nprepend_path(\"PATH\",\"/project//software/tensorflow/2.16/bin/\")","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Use Your Module Files","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"To expose your module files you need to tell the module system where to look using the ","type":"text"},{"text":"module use ","type":"text","marks":[{"type":"code"}]},{"text":" command.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[]$ module avail\n-------------- /apps/s/lmod/mf/opt/linux-rhel9-x86_64/containers ---------------\n stress-ng/0.17.08\n...","type":"text"}]},{"type":"codeBlock","content":[{"text":"# Expose your module files.\n[]$ module use /project//software/modules/","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module avail\n------------------------ /project//software/modules ------------------------\n tensorflow/2.16\n-------------- /apps/s/lmod/mf/opt/linux-rhel9-x86_64/containers ---------------\n stress-ng/0.17.08\n...","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module spider tensorflow\n----------------------------------------------------------------------------\n tensorflow: tensorflow/2.16\n----------------------------------------------------------------------------\n This module can be loaded directly: module load tensorflow/2.16","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Using Your Module","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Loading a local module has the same effect as using a System module.","type":"text"}]},{"type":"paragraph","content":[{"text":"Your environment will be updated by setting appropriate environment variables.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[@mblog1]$ salloc -A -t 10:00\nsalloc: Granted job allocation 845851\nsalloc: Nodes mbcpu-001 are ready for job\n[@mbcpu-001]$ module use /project//software/modules/\n[@mbcpu-001]$ module load tensorflow/2.16\n[@mbcpu-001]$ python --version\nPython 3.11.9\n\n[@mbcpu-001]$ which python\n/project//software/tensorflow/2.16/bin/python","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Notice the sysconfig","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"After loading our module and exposing the conda environment, notice how the ","type":"text"},{"text":"sysconfig","type":"text","marks":[{"type":"code"}]},{"text":" is based around this environment.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[@mbcpu-001]$ python -m sysconfig\nPlatform: \"linux-x86_64\"\nPython version: \"3.11\"\nCurrent installation scheme: \"posix_prefix\"\nPaths:\n data = \"/project//software/tensorflow/2.16\"\n ...\n stdlib = \"/project//software/tensorflow/2.16/lib/python3.11\"\nVariables:\n ...\n userbase = \"/cluster/medbow/project//software/tensorflow/2.16\"","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Notice that the ","type":"text"},{"text":"userbase","type":"text","marks":[{"type":"code"}]},{"text":" is pointing to where our conda environment was installed.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Running our TF Code","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"All the conda and pip package installs within this conda environment are available.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[@mbcpu-001]$ python -c \"import tensorflow as tf; print(\\\"TensorFlow Version: \\\" + str( tf.__version__))\"\n...\nTensorFlow Version: 2.16.1\n[@mbcpu-001]$ exit\nexit\nsalloc: Relinquishing job allocation 845851","type":"text"}]},{"type":"rule"},{"type":"table","attrs":{"layout":"default","width":1011.0,"localId":"f43d2270-2528-4705-8f9c-39e7644f03c6"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[333.0]},"content":[{"type":"paragraph","content":[{"text":"Prev","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Create a Shared TensorFlow Conda Environment","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2182447159"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[326.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Workshop Home","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Using Python, Conda and Pip on the Cluster","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2136342529"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[349.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Next","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Extend Conda Environment to Jupyter Kernel","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2237988917"}}]}]}]}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]}],"version":1}

Browser not supported