{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Goal","type":"text","marks":[{"type":"strong"}]},{"text":": Demonstrate extending a Python related conda environment in to Juypter kernel.","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"1"},"outline":{"value":"false"},"style":{"value":"none"},"type":{"value":"list"},"printable":{"value":"true"}},"macroMetadata":{"macroId":{"value":"b55f18c3-ba7c-4b74-9483-19dfd0f955be"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"3a3bc0d3-9b47-4e06-9dd8-598668e30590"}},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"What is Jupyter and What is a Kernel?","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Jupyter","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://docs.jupyter.org/en/latest/index.html"}}]},{"text":" ","type":"text"},{"text":"is a large umbrella project that covers many different software offerings and tools, including the popular ","type":"text","marks":[{"type":"em"}]},{"text":"Jupyter Notebook","type":"text","marks":[{"type":"em"},{"type":"link","attrs":{"href":"https://jupyter-notebook.readthedocs.io/en/latest/"}}]},{"text":" and ","type":"text","marks":[{"type":"em"}]},{"text":"JupyterLab","type":"text","marks":[{"type":"em"},{"type":"link","attrs":{"href":"https://jupyterlab.readthedocs.io/en/latest/"}}]},{"text":" ","type":"text","marks":[{"type":"em"}]},{"text":"web-based notebook authoring and editing applications","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kernels","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://docs.jupyter.org/en/latest/projects/kernels.html#kernels"}}]},{"text":" are ","type":"text"},{"text":"programming language specific ","type":"text","marks":[{"type":"em"}]},{"text":"processes","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":" that run independently and interact with the Jupyter Applications and their user interfaces.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"They can be considered as the ","type":"text"},{"text":"computational engine for notebooks","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"kernel process","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://hex.tech/blog/jupyter-kernel-overview/"}}]},{"text":" is a standalone process that runs in the background and executes the code that you write in your notebooks. The kernel process is responsible for running the code and returning the results to the frontend.","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ARCC provides the ","type":"text"},{"text":"Jupyter with OnDemand","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2205646897"}}]},{"text":" workshop.","type":"text"}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"General Process","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The general process involves updating the conda environment to include kernel related packages, and then configuring the kernel spec to allow it to be picked up by the Jupyter service.","type":"text"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Activate you Python Conda Environment.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Conda install the ","type":"text"},{"text":"ipykernel","type":"text","marks":[{"type":"code"}]},{"text":" package.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Deactivate your Conda environment.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Copy the created ","type":"text"},{"text":"kernelspec","type":"text","marks":[{"type":"code"}]},{"text":" into your home ","type":"text"},{"text":".local/share/jupyter/kernels/","type":"text","marks":[{"type":"code"}]},{"text":" folder.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update the kernel.json.","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"The process creates a kernel for an individual user from the previously created shared Conda environment.","type":"text"}]},{"type":"paragraph","content":[{"text":"Each individual from the shared space will need to follow this process to create their own kernel.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"If anyone updated the packages within this shared Conda environment, then the update will effect everyone. Be warned.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Install the ","type":"text"},{"text":"ipykernel","type":"text","marks":[{"type":"code"}]},{"text":" package","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module purge\n[]$ module load miniconda3/24.3.0\n[]$ conda activate /cluster/medbow/project//software/tensorflow/2.16\n(/cluster/medbow/project//software/tensorflow/2.16) []$ export PYTHONUSERBASE=$CONDA_PREFIX\n(/cluster/medbow/project//software/tensorflow/2.16) []$ conda install ipykernel\n...\n(/cluster/medbow/project//software/tensorflow/2.16) []$ python -m ipykernel install --user --name=TF2.16-local\n\n(/cluster/medbow/project//software/tensorflow/2.16) []$ conda deactivate\n[]$ ","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"The Related Kernel Details","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Using the ","type":"text"},{"text":"python -m ipykernel install --user --name=TF2.16-local","type":"text","marks":[{"type":"code"}]},{"text":" above will result in the following output of the form:","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"() (/project//software/tensorflow/2.16) []$ python -m ipykernel install --user --name=TF2.16-local\nInstalled kernelspec TF2.16-local in /cluster/medbow/home//.local/share/jupyter/kernels/tf2.16-local","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Notice","type":"text","marks":[{"type":"strong"}]},{"text":": This file is created in the user's ","type":"text"},{"text":"home","type":"text","marks":[{"type":"code"}]},{"text":" folder, and can be accessed and viewed by the user:","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[]$ ls /cluster/medbow/home//.local/share/jupyter/kernels/tf2.16-local/\nkernel.json logo-32x32.png logo-64x64.png logo-svg.svg\n\n[]$ cat /cluster/medbow/home//.local/share/jupyter/kernels/tf2.16-local/kernel.json\n{\n \"argv\": [\n \"/project//software/tensorflow/2.16/bin/python\",\n \"-Xfrozen_modules=off\",\n \"-m\",\n \"ipykernel_launcher\",\n \"-f\",\n \"{connection_file}\"\n ],\n \"display_name\": \"TF2.16-local\",\n \"language\": \"python\",\n \"metadata\": {\n \"debugger\": true\n }\n}","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Suggestion: Using “","type":"text"},{"text":"-local","type":"text","marks":[{"type":"code"}]},{"text":"” will help to distinguish that this is a local kernel, based on a Conda environment, that the user has created, and not something provided by ARCC.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Side Note","type":"text","marks":[{"type":"strong"}]},{"text":": Installing the ","type":"text"},{"text":"ipykernel","type":"text","marks":[{"type":"code"}]},{"text":" package creates a kernel spec related folder within the Conda environment, under: ","type":"text"},{"text":"share/jupyter/kernels/","type":"text","marks":[{"type":"code"}]},{"text":" named ","type":"text"},{"text":"python3","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"Running ","type":"text"},{"text":"python -m ipykernel","type":"text","marks":[{"type":"code"}]},{"text":" will copy and update this folder.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Kernel names can only contain ASCII letters and numbers and these separators: - . _ (hyphen, period, and underscore)","type":"text"}]},{"type":"paragraph","content":[{"text":"Names can not use whitespace such as ","type":"text"},{"text":"--name=\"TF2.16 (local)\"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Start Jupyter","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"From OnDemand start a Jupyter session.","type":"text"}]},{"type":"paragraph","content":[{"text":"Notice how the newly configured kernel is now available.","type":"text"}]}]},{"type":"mediaSingle","attrs":{"layout":"center","width":438,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":438,"alt":"image-20240826-180859.png","id":"81d9c111-6c74-4f54-b7aa-f66b0f187976","collection":"contentId-2237988917","type":"file","height":164}},{"type":"caption","content":[{"text":"Notice the new TF2.16 kernel.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Within a Notebook","type":"text"}]},{"type":"paragraph","content":[{"text":"Within a Jupyter notebook cell try:","type":"text"}]},{"type":"codeBlock","content":[{"text":"import tensorflow as tf; \nprint(\"TensorFlow Version: \" + str( tf.__version__))","type":"text"}]},{"type":"codeBlock","content":[{"text":"TensorFlow Version: 2.16.1","type":"text"}]},{"type":"rule"},{"type":"table","attrs":{"layout":"default","width":1011.0,"localId":"f43d2270-2528-4705-8f9c-39e7644f03c6"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[333.0]},"content":[{"type":"paragraph","content":[{"text":"Prev","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Create a Module File to Load Your Conda Environment","type":"text","marks":[{"type":"underline"},{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2181923764"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[326.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Workshop Home","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Using Python, Conda and Pip on the Cluster","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2136342529"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[349.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Next","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Jupyter Python Packages and Issues","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2250113039"}}]}]}]}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Add label","type":"text"}]}],"version":1}

Browser not supported