vortexSetupScripts/env_setup.sh at main · aliciayuting/vortexSetupScripts · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/bin/bash

# Check if the OPT_HOME argument is provided
if [ -z "$1" ]; then
    echo "your USER_HOME directory not provided. Please provide it as the first argument."
    echo "you can run this script as $./env_setup.sh /home/[yournetid] or $./env_setup.sh ~"
    return 1
fi

# Set OPT_HOME to the first argument
USER_HOME=$(echo "$1" | sed 's:/*$::')
OPT_HOME=${USER_HOME}/opt-dev

# Create opt-dev directory if it does not exist
if [ ! -d "$OPT_HOME" ]; then
    echo "Creating $OPT_HOME directory..."
    mkdir -p "$OPT_HOME"
    echo "Directory created: $OPT_HOME"
else
    echo "$OPT_HOME already exists. Skipping creation."
fi

export CMAKE_INSTALL_PREFIX=${USER_HOME}/.local
export CASCADE_INSTALL_PREFIX=${OPT_HOME}
export DERECHO_INSTALL_PREFIX=${OPT_HOME}
export VORTEX_INSTALL_PREFIX=${OPT_HOME}

export SPDLOG_PREFIX=${OPT_HOME}
export FAISS_INSTALL_PREFIX=${OPT_HOME}
export CURL_INSTALL_PREFIX=${OPT_HOME}/
export HNSWLIB_INSTALL_PREFIX=${OPT_HOME}/
export CMAKE_PREFIX_PATH=${OPT_HOME}/
export C_INCLUDE_PATH=${OPT_HOME}/include/:${C_INCLUDE_PATH}
export CPLUS_INCLUDE_PATH=${OPT_HOME}/include/
export LIBRARY_PATH=${OPT_HOME}/lib/:${LIBRARY_PATH}
export LD_LIBRARY_PATH=${OPT_HOME}/lib/
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
export CUDAToolKitRoot=/usr/local/cuda-12.3
export PYTHONPATH=$PYTHONPATH:${OPT_HOME}/lib/
export PYTHONPATH=$PYTHONPATH:${OPT_HOME}/bin/
export PATH=${USER_HOME}/.local/bin:$PATH
export PATH=${OPT_HOME}/bin:$PATH
export PATH=/usr/local/cuda/bin:$PATH

# After installing pybind via pip install --user pybind11
export pybind11_DIR=${USER_HOME}/.local/lib/python3.10/site-packages/pybind11/share/cmake/pybind11

# To fix the UDL numpy import issue
export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libpython3.10.so
export PYTHONPATH=${USER_HOME}/.local/lib/python3.10/site-packages/:$PYTHONPATH

# # Fractus location of huggingface cache and models
# sudo mkdir -p /mnt/vortex/models/.huggingface_cache
# # Create a group named vortex if it doesn't exist
# MYGROUP=vortex
# getent group "$MYGROUP" >/dev/null || sudo groupadd "$MYGROUP"
# # Add users to the shared group
# sudo usermod -aG "$MYGROUP" yy354
# sudo usermod -aG "$MYGROUP" az275
# # Set group ownership and permissions (setgid so new files inherit group)
# sudo chgrp -R "$MYGROUP" /mnt/vortex/models/.huggingface_cache
# sudo chmod -R 2775 /mnt/vortex/models/.huggingface_cache
# # Optional: ensure group write on existing files
# sudo chmod -R g+rwX /mnt/vortex/models/.huggingface_cache
# echo "Shared Hugging Face cache ready for group: $MYGROUP"

export HF_HOME=/mnt/vortex/models/.huggingface_cache
echo "set HF_HOME to /mnt/vortex/models/.huggingface_cache"


echo "run this script again after install FLMR"
export PYTHONPATH=~/workspace/FLMR:$PYTHONPATH

echo "Environment variables have been set with OPT_HOME: $OPT_HOME"