Versions Compared

Old Version 8

changes.mady.by.user Simon Alexander

Saved on

compared with

New Version 9

changes.mady.by.user Simon Alexander

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

Info

To help demonstrate the various concepts, lets create the following environment:

Code Block
[salexan5@mblog2 ~] cd /project/arcc<project-name>/salexan5<username>/conda
[salexan5@mblog2 conda]$ module load miniconda3/24.3.0
[salexan5@mblog2 conda]$ conda create -p py_conda_pip_env python=3.12
[salexan5@mblog2 conda]$ conda activate /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ python --version
Python 3.12.4
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ which python
/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/bin/python
# Create a self contined environment i.e don't install anything into the local home folder.
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ export PYTHONUSERBASE=$CONDA_PREFIX
# Use conda to install: numpy
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ conda install numpy
# Use pip to install: nltk
# has dependencies: click, joblib, regex and tqdm
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ pip install nltk

...

What is Installed? conda list

...

Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ conda list
...
click                     8.1.7                    pypi_0    pypi
joblib                    1.4.2                    pypi_0    pypi
nltk                      3.8.1                    pypi_0    pypi
numpy                     2.0.1           py312h1103770_0    conda-forge
regex                     2024.7.24                pypi_0    pypi
tqdm                      4.66.4                   pypi_0    pypi
Expand
titlefull conda list
Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ conda list
# packages in environment at /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env:
#
# Name                    Version                   Build  Channel
_libgcc_mutex             0.1                 conda_forge    conda-forge
_openmp_mutex             4.5                       2_gnu    conda-forge
bzip2                     1.0.8                h4bc722e_7    conda-forge
ca-certificates           2024.7.4             hbcca054_0    conda-forge
click                     8.1.7                    pypi_0    pypi
joblib                    1.4.2                    pypi_0    pypi
ld_impl_linux-64          2.40                 hf3520f5_7    conda-forge
libblas                   3.9.0           23_linux64_openblas    conda-forge
libcblas                  3.9.0           23_linux64_openblas    conda-forge
libexpat                  2.6.2                h59595ed_0    conda-forge
libffi                    3.4.2                h7f98852_5    conda-forge
libgcc-ng                 14.1.0               h77fa898_0    conda-forge
libgfortran-ng            14.1.0               h69a702a_0    conda-forge
libgfortran5              14.1.0               hc5f4f2c_0    conda-forge
libgomp                   14.1.0               h77fa898_0    conda-forge
liblapack                 3.9.0           23_linux64_openblas    conda-forge
libnsl                    2.0.1                hd590300_0    conda-forge
libopenblas               0.3.27          pthreads_hac2b453_1    conda-forge
libsqlite                 3.46.0               hde9e2c9_0    conda-forge
libstdcxx-ng              14.1.0               hc0a3c3a_0    conda-forge
libuuid                   2.38.1               h0b41bf4_0    conda-forge
libxcrypt                 4.4.36               hd590300_1    conda-forge
libzlib                   1.3.1                h4ab18f5_1    conda-forge
ncurses                   6.5                  h59595ed_0    conda-forge
nltk                      3.8.1                    pypi_0    pypi
numpy                     2.0.1           py312h1103770_0    conda-forge
openssl                   3.3.1                h4bc722e_2    conda-forge
pip                       24.0               pyhd8ed1ab_0    conda-forge
python                    3.12.4          h194c7f8_0_cpython    conda-forge
python_abi                3.12                    4_cp312    conda-forge
readline                  8.2                  h8228510_1    conda-forge
regex                     2024.7.24                pypi_0    pypi
setuptools                71.0.4             pyhd8ed1ab_0    conda-forge
tk                        8.6.13          noxft_h4845f30_101    conda-forge
tqdm                      4.66.4                   pypi_0    pypi
tzdata                    2024a                h0c530f3_0    conda-forge
wheel                     0.43.0             pyhd8ed1ab_1    conda-forge
xz                        5.2.6                h166bdaf_0    conda-forge

...

Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ pip list -v
Package    Version   Location                                                                                  Installer
---------- --------- ----------------------------------------------------------------------------------------- ---------
click      8.1.7     /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages pip
joblib     1.4.2     /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages pip
nltk       3.8.1     /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages pip
numpy      2.0.1     /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages conda
pip        24.0      /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages
regex      2024.7.24 /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages pip
setuptools 71.0.4    /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages
tqdm       4.66.4    /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages pip
wheel      0.43.0    /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages

...

Expand
titleList what's in: /project/arcc/salexan5/conda/py_conda_pip_env/lib/python3.12/site-packages/
Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ ls /project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env/lib/python3.12/site-packages/
click                     joblib                  numpy                     pkg_resources              setuptools                         wheel
click-8.1.7.dist-info     joblib-1.4.2.dist-info  numpy-2.0.1.dist-info     README.txt                 setuptools-71.0.4-py3.12.egg-info  wheel-0.43.0.dist-info
_distutils_hack           nltk                    pip                       regex                      tqdm
distutils-precedence.pth  nltk-3.8.1.dist-info    pip-24.0-py3.12.egg-info  regex-2024.7.24.dist-info  tqdm-4.66.4.dist-info

...

Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ conda env export > env.yml
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ cat env.yml
name: /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env
...
dependencies:
...
  - numpy=2.0.1=py312h1103770_0
...
  - python=3.12.4=h194c7f8_0_cpython
...
  - pip:
      - click==8.1.7
      - joblib==1.4.2
      - nltk==3.8.1
      - regex==2024.7.24
      - tqdm==4.66.4
prefix: /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env
Expand
titlefull output:
Code Block
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ conda env export > env.yml
(/cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env) [salexan5@mblog2 conda]$ cat env.yml
name: /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env
channels:
  - conda-forge
  - defaults
dependencies:
  - _libgcc_mutex=0.1=conda_forge
  - _openmp_mutex=4.5=2_gnu
  - bzip2=1.0.8=h4bc722e_7
  - ca-certificates=2024.7.4=hbcca054_0
  - ld_impl_linux-64=2.40=hf3520f5_7
  - libblas=3.9.0=23_linux64_openblas
  - libcblas=3.9.0=23_linux64_openblas
  - libexpat=2.6.2=h59595ed_0
  - libffi=3.4.2=h7f98852_5
  - libgcc-ng=14.1.0=h77fa898_0
  - libgfortran-ng=14.1.0=h69a702a_0
  - libgfortran5=14.1.0=hc5f4f2c_0
  - libgomp=14.1.0=h77fa898_0
  - liblapack=3.9.0=23_linux64_openblas
  - libnsl=2.0.1=hd590300_0
  - libopenblas=0.3.27=pthreads_hac2b453_1
  - libsqlite=3.46.0=hde9e2c9_0
  - libstdcxx-ng=14.1.0=hc0a3c3a_0
  - libuuid=2.38.1=h0b41bf4_0
  - libxcrypt=4.4.36=hd590300_1
  - libzlib=1.3.1=h4ab18f5_1
  - ncurses=6.5=h59595ed_0
  - numpy=2.0.1=py312h1103770_0
  - openssl=3.3.1=h4bc722e_2
  - pip=24.0=pyhd8ed1ab_0
  - python=3.12.4=h194c7f8_0_cpython
  - python_abi=3.12=4_cp312
  - readline=8.2=h8228510_1
  - setuptools=71.0.4=pyhd8ed1ab_0
  - tk=8.6.13=noxft_h4845f30_101
  - tzdata=2024a=h0c530f3_0
  - wheel=0.43.0=pyhd8ed1ab_1
  - xz=5.2.6=h166bdaf_0
  - pip:
      - click==8.1.7
      - joblib==1.4.2
      - nltk==3.8.1
      - regex==2024.7.24
      - tqdm==4.66.4
prefix: /cluster/medbow/project/arcc<project-name>/salexan5<username>/conda/py_conda_pip_env

...

Prev

Jupyter Python Packages and Issues

Workshop Home

Using Python, Conda and Pip on the Cluster

Next

Python, Conda and Pip: Suggested Best Practices

...