From ec9517c0c9848c00ab8f91b1d9ca97c7ef6e4019 Mon Sep 17 00:00:00 2001
From: Mathieu Broillet <mathieu@broillet.ch>
Date: Tue, 27 Aug 2024 17:44:25 +0200
Subject: [PATCH] update text-gen

---
 llama-cpp-python-rocm-build/Dockerfile |  2 +-
 text-generation-rocm/install.sh        | 30 +++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/llama-cpp-python-rocm-build/Dockerfile b/llama-cpp-python-rocm-build/Dockerfile
index ebcf3ab..e8dd209 100644
--- a/llama-cpp-python-rocm-build/Dockerfile
+++ b/llama-cpp-python-rocm-build/Dockerfile
@@ -21,7 +21,7 @@ ENV VIRTUAL_ENV=/opt/venv
 RUN python3.10 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN pip3 install --upgrade pip wheel setuptools
+RUN pip3 install --upgrade pip wheel setuptools build
 
 # Install pytorch for rocm
 RUN pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/${TORCH_VERSION}
diff --git a/text-generation-rocm/install.sh b/text-generation-rocm/install.sh
index a92cf18..9201d1a 100755
--- a/text-generation-rocm/install.sh
+++ b/text-generation-rocm/install.sh
@@ -16,15 +16,38 @@ install() {
 
     else
         # Add BnB
-        $python_exec -m pip install --upgrade ../bitsandbytes-rocm-build/bitsandbytes-0.43.3.dev0-cp310-cp310-linux_x86_64.whl # install bitsandbytes for rocm until it is available on pypi
+        $python_exec -m pip install --upgrade https://github.com/M4TH1EU/ai-suite-rocm-local/releases/download/prebuilt-wheels-for-rocm/bitsandbytes-0.43.3-cp310-cp310-linux_x86_64.whl # install bitsandbytes for rocm until it is available on pypi
 
         # Add AutoGPTQ
-        $python_exec -m pip install auto-gptq[triton] --no-build-isolation --extra-index-url https://huggingface.github.io/autogptq-index/whl/rocm573/
+        $python_exec -m pip install auto-gptq --no-build-isolation --extra-index-url https://huggingface.github.io/autogptq-index/whl/rocm573/
+
+        # Add ExLlamav2
+        $python_exec -m pip install https://github.com/turboderp/exllamav2/releases/download/v0.1.9/exllamav2-0.1.9+rocm6.1.torch2.4.0-cp310-cp310-linux_x86_64.whl
 
         # Add LlamaCPP
-        CMAKE_ARGS="-DGGML_HIPBLAS=on" FORCE_CMAKE=1 $python_exec -m pip install llama-cpp-python
+        CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python # cpu
+        # CMAKE_ARGS="-DGGML_HIPBLAS=on" FORCE_CMAKE=1 $python_exec -m pip install llama-cpp-python # gpu
+
+        # llama cpp built with hipblas doesn't work unless the whole rocm stack is installed locally
+        # so for now, use llama with openblas (cpu)
+
+#        main_venv_path=$(dirname $(python -c "import torch; print(torch.__file__)"))"/lib/"
+#        llama_lib_path="$(pwd)/venv/lib64/python3.10/site-packages/llama_cpp/lib"
+#
+#        for file in "$main_venv_path"/*.so; do
+#            ln -s "$file" "$llama_lib_path/$(basename "$file")"
+#        done
+
+#        ln -s "$llama_lib_path/libhipblas.so" "$llama_lib_path/libhipblas.so.1"
+#        ln -s "$llama_lib_path/libhipblas.so" "$llama_lib_path/libhipblas.so.2"
+#        ln -s "$llama_lib_path/librocblas.so" "$llama_lib_path/librocblas.so.3"
+#        ln -s "$llama_lib_path/librocblas.so" "$llama_lib_path/librocblas.so.4"
+#        ln -s "$llama_lib_path/libamdhip64.so" "$llama_lib_path/libamdhip64.so.5"
+#        ln -s "$llama_lib_path/libamdhip64.so" "$llama_lib_path/libamdhip64.so.6"
+
 
         # Add Triton
+#        $python_exec -m pip install https://github.com/M4TH1EU/ai-suite-rocm-local/releases/download/prebuilt-wheels-for-rocm/llama_cpp_python-0.2.89-cp310-cp310-linux_x86_64.whl
 #        git clone https://github.com/ROCmSoftwarePlatform/triton.git .tritonrocm
 #        cd .tritonrocm/python
 #        $python_exec -m pip install ninja cmake; # build time dependencies
@@ -41,6 +64,7 @@ install() {
         sed -i '/lm_eval/d' webui/requirements_amd.txt
         sed -i '/optimum/d' webui/requirements_amd.txt
         sed -i '/autoawq/d' webui/requirements_amd.txt
+        sed -i '/llama_cpp_python/d' webui/requirements_amd.txt
 
         $python_exec -m pip install -r webui/requirements_amd.txt