add llamacpp build

bitsandbytes-rocm-build/README.md

@@ -12,5 +12,4 @@ This creates a Docker image that builds the package and extract the built wheel
 
 The wheel file will be in a folder named ``build_output/``
 
-*You might also find one of my already built wheel in this folder (may not be up to date)*
-
+*You might also find one of my already built wheel in this folder or on the github releases page (may not be up to date)*

llama-cpp-python-rocm-build/.gitignore

@@ -0,0 +1 @@
+build_output/

llama-cpp-python-rocm-build/Dockerfile

@@ -0,0 +1,39 @@
+FROM rocm/dev-ubuntu-22.04:6.1.2
+
+ENV TORCH_VERSION="rocm6.1" \
+    DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8 \
+    # for llama
+    CMAKE_ARGS="-DGGML_HIPBLAS=on" \
+    FORCE_CMAKE=1
+
+WORKDIR /tmp
+
+RUN apt-get update -y
+RUN apt-get install -y wget git cron cmake make software-properties-common
+
+# Install python3.10
+RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update -y
+RUN apt-get install -y python3.10 python3.10-dev python3.10-venv
+
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3.10 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+RUN pip3 install --upgrade pip wheel setuptools
+
+# Install pytorch for rocm
+RUN pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/${TORCH_VERSION}
+
+# ROCM llama-cpp-python
+RUN apt-get install -y hipblas hipblaslt hiprand hipsparse hipcub rocthrust-dev
+## Clone repo and install python requirements
+RUN git clone --recurse-submodules https://github.com/abetlen/llama-cpp-python.git
+WORKDIR /tmp/llama-cpp-python
+
+## Build
+RUN python3.10 -m build --wheel
+
+# Cleanup
+RUN apt-get clean && pip3 cache purge

llama-cpp-python-rocm-build/README.md

@@ -0,0 +1,16 @@
+# llama-cpp-python-rocm-build-pip
+
+This is a simple script to help you build the latest llama-cpp-python for rocm.
+The official build process requires a lot of ROCM-related packages and can mess up your computer easily if you install the wrong packages.
+
+This creates a Docker image that builds the package and extract the built wheel file that you can then easily install using pip.
+
+```bash
+./build.sh
+./extract_build.sh
+```
+
+The wheel file will be in a folder named ``build_output/``
+
+*You might also find one of my already built wheel in this folder or on the github releases page (may not be up to date)*
+

llama-cpp-python-rocm-build/build.sh

@@ -0,0 +1 @@
+docker build . -t 'llama-cpp-python-rocm-build:6.1.2' -f Dockerfile

llama-cpp-python-rocm-build/extract_build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Set variables
+IMAGE_NAME="llama-cpp-python-rocm-build:6.1.2"
+CONTAINER_NAME="llama-cpp-python-rocm-build"
+FILE_IN_CONTAINER="/tmp/llama-cpp-python/dist/"
+FILE_ON_HOST="./build_output/"
+
+# Run the Docker container
+docker run -d --name $CONTAINER_NAME $IMAGE_NAME
+
+# Check if the container is running
+if [ "$(docker ps -q -f name=$CONTAINER_NAME)" ]; then
+    echo "Container $CONTAINER_NAME is running."
+
+    # Copy the file from the container to the host
+    docker cp $CONTAINER_NAME:$FILE_IN_CONTAINER $FILE_ON_HOST
+
+    if [ $? -eq 0 ]; then
+        echo "File copied successfully to $FILE_ON_HOST"
+    else
+        echo "Failed to copy file."
+    fi
+else
+    echo "Failed to start container $CONTAINER_NAME."
+fi
+
+docker stop $CONTAINER_NAME
+docker rm $CONTAINER_NAME
+
+echo "Now you can install llama-cpp-python locally using \"pip install\" with the file in the build_output/ folder"

text-generation-rocm/install.sh

@@ -22,8 +22,7 @@ install() {
         $python_exec -m pip install auto-gptq[triton] --no-build-isolation --extra-index-url https://huggingface.github.io/autogptq-index/whl/rocm573/
 
         # Add LlamaCPP
-        CMAKE_ARGS="-DLLAMA_CLBLAST=on" FORCE_CMAKE=1 $python_exec -m pip install llama-cpp-python
-        # CMAKE_ARGS="-DGGML_HIPBLAS=on" FORCE_CMAKE=1 $python_exec -m pip install llama-cpp-python
+        CMAKE_ARGS="-DGGML_HIPBLAS=on" FORCE_CMAKE=1 $python_exec -m pip install llama-cpp-python
 
         # Add Triton
 #        git clone https://github.com/ROCmSoftwarePlatform/triton.git .tritonrocm
@@ -45,10 +44,11 @@ install() {
 
         $python_exec -m pip install -r webui/requirements_amd.txt
 
-        $python_exec -m pip install accelerate # only works after requirements_amd.txt is installed ??!
-        $python_exec -m pip install lm_eval optimum autoawq
+        # only works after requirements_amd.txt is installed ??!
+        $python_exec -m pip install accelerate optimum
         $python_exec -m pip install https://github.com/casper-hansen/AutoAWQ_kernels/releases/download/v0.0.7/autoawq_kernels-0.0.7+rocm571-cp310-cp310-linux_x86_64.whl --no-deps
         $python_exec -m pip install https://github.com/casper-hansen/AutoAWQ/releases/download/v0.2.6/autoawq-0.2.6-cp310-cp310-linux_x86_64.whl --no-deps
+        $python_exec -m pip install lm_eval
         ln -s webui/models models
     fi
 

text-generation-rocm/run.sh

@@ -10,8 +10,9 @@ main() {
     # Prints ROCM info with available GPUs
     rocm-smi
 
-    # Start SD
-    TORCH_BLAS_PREFER_HIPBLASLT=0 $python_exec webui/server.py --listen
+    # Start
+    cd webui
+    TORCH_BLAS_PREFER_HIPBLASLT=0 $python_exec server.py --listen
 }
 
 main