diff --git a/.github/workflows/build-and_publish-prebuilts-whl.yml b/.github/workflows/build-and_publish-prebuilts-whl.yml
index 7e69869..16bc2f0 100644
--- a/.github/workflows/build-and_publish-prebuilts-whl.yml
+++ b/.github/workflows/build-and_publish-prebuilts-whl.yml
@@ -59,6 +59,12 @@ jobs:
           mkdir -p /tmp/llama-cpp-python/
           docker cp prebuilt-container:/tmp/llama-cpp-python/dist/ /tmp/llama-cpp-python/
 
+      # Step 6: Copy xformers wheel artifact to host
+      - name: Copy xformers wheel to host
+        run: |
+          mkdir -p /tmp/xformers/
+          docker cp prebuilt-container:/tmp/xformers/dist/ /tmp/xformers/
+
       # Step 7: Upload bitsandbytes wheel artifact
       - name: Upload bitsandbytes wheel
         uses: actions/upload-artifact@v4
@@ -73,7 +79,14 @@ jobs:
           name: llama-cpp-python-wheels
           path: /tmp/llama-cpp-python/dist/*.whl
 
-      # Step 9: Cleanup Docker container
+      # Step 9: Upload xformers wheel artifact
+      - name: Upload xformers wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: xformers-wheels
+          path: /tmp/xformers/dist/*.whl
+
+      # Step 10: Cleanup Docker container
       - name: Cleanup
         run: |
           docker rm prebuilt-container
\ No newline at end of file
diff --git a/prebuilts/entrypoint.sh b/prebuilts/entrypoint.sh
index 5903fb6..1822ecf 100755
--- a/prebuilts/entrypoint.sh
+++ b/prebuilts/entrypoint.sh
@@ -15,4 +15,12 @@ python3.10 setup.py bdist_wheel --universal
 ## Clone repo and install python requirements
 git clone --recurse-submodules https://github.com/abetlen/llama-cpp-python.git /tmp/llama-cpp-python
 cd /tmp/llama-cpp-python
-CMAKE_ARGS="-D GGML_HIPBLAS=on -D AMDGPU_TARGETS=${GPU_TARGETS}" FORCE_CMAKE=1 python3.10 -m build --wheel
\ No newline at end of file
+CMAKE_ARGS="-D GGML_HIPBLAS=on -D AMDGPU_TARGETS=${GPU_TARGETS}" FORCE_CMAKE=1 python3.10 -m build --wheel
+
+
+# ROCM xformers
+## Clone repo and install python requirements
+pip3 install ninja
+git clone --depth 1 https://github.com/facebookresearch/xformers.git /tmp/xformers
+cd /tmp/xformers
+python3.10 setup.py bdist_wheel --universal
\ No newline at end of file