{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Goal","type":"text","marks":[{"type":"strong"}]},{"text":": Understand where pip installs packages within a user’s home folder with respect to different versions of Python.","type":"text"}]},{"type":"rule"},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"style":{"value":"none"}},"macroMetadata":{"macroId":{"value":"5cbcd74e-0e2b-4bc5-8f3b-7b353b74a36e"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"aca13525-88a6-455d-bfcb-49455fcda37a"}},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Python on the System","type":"text"}]},{"type":"paragraph","content":[{"text":"Python is available on the System (but not necessarily on the compute nodes due to keeping node images ","type":"text"},{"text":"lite","type":"text","marks":[{"type":"em"}]},{"text":").","type":"text"}]},{"type":"codeBlock","content":[{"text":"[salexan5@mblog2 ~]$ python --version\nPython 3.9.18\n\n[salexan5@mblog1 ~]$ which python\n/usr/bin/python","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Version might update during cluster maintenance.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Pip may/may","type":"text"},{"text":" not","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"0b040885-9e1c-43eb-ba2e-a192c80f3a72"}}]},{"text":" be installed.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Recommended to not use to due potential updates.","type":"text"}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Python as a Module","type":"text"}]},{"type":"paragraph","content":[{"text":"Different versions are available via loading a module","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module spider python\n----------------------------------------------------------------------------\n python:\n----------------------------------------------------------------------------\n Versions:\n python/3.10.6\n python/3.12.0\n Other possible modules matches:\n python2\n\n[]$ module purge \n[]$ module load gcc/14.2.0 python/3.10.6\n[]$ python --version\nPython 3.10.6\n[]$ which pip\n/apps/u/spack/gcc/14.2.0/python/3.10.6-6lvrsdd/bin/pip ","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Recommended to use a module version for reproducibility.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Pip will be available.","type":"text"}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Pip Install Numpy - Python 3.10.6","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Let’s ","type":"text"},{"text":"pip install numpy","type":"text","marks":[{"type":"code"}]},{"text":" and then run a test to verify that it works.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"# Using python/3.10.6\n[]$ pip install numpy\nDefaulting to user installation because normal site-packages is not writeable\nCollecting numpy\n Downloading numpy-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\n ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 60.9/60.9 kB 2.0 MB/s eta 0:00:00\nDownloading numpy-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (19.3 MB)\n ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 19.3/19.3 MB 131.3 MB/s eta 0:00:00\nInstalling collected packages: numpy\nSuccessfully installed numpy-2.0.0\n\n[notice] A new release of pip is available: 24.0 -> 24.1\n[notice] To update, run: pip install --upgrade pip","type":"text"}]},{"type":"expand","attrs":{"title":"py_test.py"},"content":[{"type":"codeBlock","content":[{"text":"import sys\nimport numpy\nprint(\"Python version: \" + sys.version)\nprint(\"Numpy: \" + str(numpy.version.version))","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"[]$ python py_test.py\nPython version: 3.10.6 (main, Sep 3 2024, 15:13:56) [GCC 14.2.0]\nNumpy: 2.0.0","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Use Python 3.12.0 - Is numpy available?","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module purge\n[]$ module load gcc/14.2.0 python/3.12.0\n[]$ python --version\nPython 3.12.0\n[]$ which python\n/apps/u/spack/gcc/14.2.0/python/3.12.0-4e5he6r/bin/python\n\n[]$ python py_test.py\nTraceback (most recent call last):\n File \"/cluster/medbow/home//py_test.py\", line 2, in \n import numpy\nModuleNotFoundError: No module named 'numpy","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"But didn’t we previously install the ","type":"text"},{"text":"numpy","type":"text","marks":[{"type":"code"}]},{"text":" package?","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Python System Configuration","type":"text"}]},{"type":"paragraph","content":[{"text":"sysconfig","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.python.org/3/library/sysconfig.html#module-sysconfig"}}]},{"text":": The ","type":"text"},{"text":"sysconfig","type":"text","marks":[{"type":"code"}]},{"text":" module provides access to Python’s configuration information like the list of installation paths and the configuration variables relevant for the current platform.","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module purge\n[]$ module load gcc/14.2.0 python/3.10.6\n[]$ python -m sysconfig\nPlatform: \"linux-x86_64\"\nPython version: \"3.10\"\nCurrent installation scheme: \"posix_prefix\"\nPaths:\n ...\n\t\tscripts = \"/apps/u/spack/gcc/14.2.0/python/3.10.6-6lvrsdd/bin\"\n stdlib = \"/apps/u/spack/gcc/14.2.0/python/3.10.6-6lvrsdd/lib/python3.10\"\n...\nVariables:\n ... \n py_version = \"3.10.6\"\n ...\n py_version_short = \"3.10\"\n ...\n userbase = \"/home//.local\" \n# Full list has over 700 lines! ","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"What is the Installation Scheme?","type":"text"}]},{"type":"paragraph","content":[{"text":"Prefix scheme","type":"text","marks":[{"type":"link","attrs":{"href":"https://docs.python.org/3/library/sysconfig.html#prefix-scheme"}}]},{"text":": The “prefix scheme” is useful when you wish to use one Python installation to perform the build/install (i.e., to run the setup script), but install modules into the third-party module directory of a different Python installation (or something that looks like a different Python installation). If this sounds a trifle unusual, it is—that’s why the user and home schemes come before. However, there are at least two known cases where the prefix scheme will be useful. ","type":"text"}]},{"type":"paragraph","content":[{"text":"posix_prefix","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"table","attrs":{"layout":"align-start","width":937.0,"localId":"fc99620c-9680-4d32-985b-c1fd12eeb5c7"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"text":"Path","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[752.0]},"content":[{"type":"paragraph","content":[{"text":"Installation directory","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"text":"stdlib","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[752.0]},"content":[{"type":"paragraph","content":[{"text":"prefix/lib/pythonX.Y","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"text":"scripts","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[752.0]},"content":[{"type":"paragraph","content":[{"text":"prefix/bin","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"text":"data","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[752.0]},"content":[{"type":"paragraph","content":[{"text":"prefix","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The prefix will be the ","type":"text"},{"text":"userbase","type":"text","marks":[{"type":"code"}]},{"text":" - so in this case: ","type":"text"},{"text":"/home//.local","type":"text","marks":[{"type":"code"}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"What is under the userbase?","type":"text"}]},{"type":"codeBlock","content":[{"text":"[~]$ ls .local\nbin lib share\n\n[]$ ls -al .local/bin\ntotal 1\ndrwxr-xr-x 2 4096 Jun 24 11:16 .\ndrwxr-xr-x 2 4096 Jun 24 11:16 ..\n-rwxr-xr-x 1 257 Jun 24 11:16 f2py\n-rwxr-xr-x 1 257 Jun 24 11:16 numpy-config\n\n[]$ ls .local/lib/\npython3.10\n[]$ ls .local/lib/python3.10/site-packages/\nnumpy numpy-2.0.0.dist-info numpy.libs\n\n# ~/.local/lib/python3.10","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Why is ","type":"text"},{"text":"there","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"5b748eb7-1a46-411e-8b89-ad27c62d360c"}}]},{"text":" no lib/python3.12?","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Because you have not pip installed anything for this ","type":"text"},{"text":"particular version","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":".","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"When using ","type":"text"},{"text":"python/3.12.0","type":"text","marks":[{"type":"code"}]},{"text":" it is looking under ","type":"text"},{"text":"~/.local/lib/python3.12","type":"text","marks":[{"type":"code"}]},{"text":" for pip installed packages.","type":"text"}]},{"type":"paragraph","content":[{"text":"Since we have not pip installed ","type":"text"},{"text":"numpy","type":"text","marks":[{"type":"code"}]},{"text":" using version 3.12.0, no package exists under this folder.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Pip Install Numpy - Python 3.12.0","type":"text"}]},{"type":"codeBlock","content":[{"text":"[]$ module purge\n[]$ module load gcc/14.2.0 python/3.12.0\n[]$ pip install numpy==1.26.4\n[]$ python py_test.py\nPython version: 3.12.0 (main, Sep 3 2024, 18:42:09) [GCC 14.2.0]\nNumpy: 1.26.4\n\n[~]$ ls .local/lib/\npython3.10 python3.12\n\n[~]$ ls .local/lib/python3.12/site-packages/\nnumpy numpy-1.26.4.dist-info numpy.libs","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"What has changed under the .local/bin folder?","type":"text"}]},{"type":"codeBlock","content":[{"text":"# Previously when installing numpy for python/3.10.6\n[~]$ ls -al .local/bin\ntotal 1\n-rwxr-xr-x 1 salexan5 salexan5 257 Jun 24 11:16 f2py\n-rwxr-xr-x 1 salexan5 salexan5 257 Jun 24 11:16 numpy-config\n[~]$ cat .local/bin/f2py\n#!/apps/u/spack/gcc/14.2.0/python/3.10.6-6lvrsdd/bin/python\n...\n\n# Now after installing numpy for python/3.12.0\n[~]$ ls -al .local/bin/\n-rwxr-xr-x 1 salexan5 salexan5 257 Jun 24 11:45 f2py\n-rwxr-xr-x 1 salexan5 salexan5 257 Jun 24 11:16 numpy-config\n[~]$ cat .local/bin/f2py\n#!/apps/u/spack/gcc/14.2.0/python/3.12.0-4e5he6r/bin/python\n...","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"python/3.10.6","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"},{"text":"numpy","type":"text","marks":[{"type":"code"}]},{"text":" related files have been overridden.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Remember: Pip Installs and Package Locations","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"In general","type":"text","marks":[{"type":"strong"}]},{"text":": Using ","type":"text"},{"text":"python/X.Y.Z","type":"text","marks":[{"type":"code"}]},{"text":" will ","type":"text"},{"text":"pip install","type":"text","marks":[{"type":"code"}]},{"text":" packages under ","type":"text"},{"text":"~/.local/lib/pythonX.Y/","type":"text","marks":[{"type":"code"}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center","width":1236,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1393,"alt":"image-20240823-142848.png","id":"ff270c70-2033-47fc-ad34-c458f49eb01f","collection":"contentId-2182676501","type":"file","height":284}}]},{"type":"rule"},{"type":"table","attrs":{"layout":"default","width":1011.0,"localId":"3460d150-7033-49c6-b917-a937ee04451d"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[333.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[326.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Workshop Home","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"Using Python, Conda and Pip on the Cluster","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2136342529"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[349.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Next","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"end"}}],"content":[{"text":"Pip Install within a Conda Environment","type":"text","marks":[{"type":"link","attrs":{"href":"https://arccwiki.atlassian.net/wiki/spaces/DOCUMENTAT/pages/2182676511"}}]}]}]}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]}],"version":1}