Skip to content

Commit 1286942

Browse files
mudlermudler
authored
feat: llama.cpp gRPC C++ backend (mudler#1170)
* wip: llama.cpp c++ gRPC server Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * make it work, attach it to the build process Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * update deps Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix: add protobuf dep Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * try fix protobuf on cmake * cmake: workarounds Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * add packages * cmake: use fixed version of grpc Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * cmake(grpc): install locally * install grpc Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * install required deps for grpc on debian bullseye Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * debug * debug * Fixups * no need to install cmake manually Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: fixup macOS * use brew whenever possible Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * macOS fixups * debug * fix container build Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * workaround * try mac https://stackoverflow.com/questions/23905661/on-mac-g-clang-fails-to-search-usr-local-include-and-usr-local-lib-by-def * Disable temp. arm64 docker image builds --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 8034ed3 commit 1286942

10 files changed

Lines changed: 1145 additions & 16 deletions

File tree

‎.github/workflows/bump_deps.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ jobs:
1212
- repository: "go-skynet/go-llama.cpp"
1313
variable: "GOLLAMA_VERSION"
1414
branch: "master"
15+
- repository: "ggerganov/llama.cpp"
16+
variable: "CPPLLAMA_VERSION"
17+
branch: "master"
1518
- repository: "go-skynet/go-ggml-transformers.cpp"
1619
variable: "GOGGMLTRANSFORMERS_VERSION"
1720
branch: "master"

‎.github/workflows/image.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ jobs:
1919
matrix:
2020
include:
2121
- build-type: ''
22-
platforms: 'linux/amd64,linux/arm64'
22+
#platforms: 'linux/amd64,linux/arm64'
23+
platforms: 'linux/amd64'
2324
tag-latest: 'auto'
2425
tag-suffix: ''
2526
ffmpeg: ''
@@ -38,7 +39,7 @@ jobs:
3839
tag-suffix: '-cublas-cuda12'
3940
ffmpeg: ''
4041
- build-type: ''
41-
platforms: 'linux/amd64,linux/arm64'
42+
platforms: 'linux/amd64'
4243
tag-latest: 'false'
4344
tag-suffix: '-ffmpeg'
4445
ffmpeg: 'true'

‎.github/workflows/release.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ jobs:
2929
run: |
3030
sudo apt-get update
3131
sudo apt-get install build-essential ffmpeg
32+
33+
git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
34+
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
35+
-DgRPC_BUILD_TESTS=OFF \
36+
../.. && sudo make -j12 install
37+
3238
- name: Build
3339
id: build
3440
env:
@@ -66,12 +72,20 @@ jobs:
6672
- uses: actions/setup-go@v4
6773
with:
6874
go-version: '>=1.21.0'
75+
- name: Dependencies
76+
run: |
77+
git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
78+
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
79+
-DgRPC_BUILD_TESTS=OFF \
80+
../.. && make -j12 install && rm -rf grpc
6981
- name: Build
7082
id: build
7183
env:
7284
CMAKE_ARGS: "${{ matrix.defines }}"
7385
BUILD_ID: "${{ matrix.build }}"
7486
run: |
87+
export C_INCLUDE_PATH=/usr/local/include
88+
export CPLUS_INCLUDE_PATH=/usr/local/include
7589
make dist
7690
- uses: actions/upload-artifact@v3
7791
with:

‎.github/workflows/test.yml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,15 @@ jobs:
6767
run: |
6868
sudo apt-get update
6969
sudo apt-get install build-essential ffmpeg
70-
70+
7171
sudo apt-get install -y ca-certificates cmake curl patch
7272
sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
7373
sudo pip install -r extra/requirements.txt
7474
75+
76+
# Pre-build stable diffusion before we install a newever version of abseil (not compatible with stablediffusion-ncn)
77+
GO_TAGS="tts stablediffusion" GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
78+
7579
sudo mkdir /build && sudo chmod -R 777 /build && cd /build && \
7680
curl -L "https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz" | \
7781
tar -xzvf - && \
@@ -87,6 +91,12 @@ jobs:
8791
sudo cp -rfv /build/lib/Linux-$(uname -m)/piper_phonemize/lib/. /usr/lib/ && \
8892
sudo ln -s /usr/lib/libpiper_phonemize.so /usr/lib/libpiper_phonemize.so.1 && \
8993
sudo cp -rfv /build/lib/Linux-$(uname -m)/piper_phonemize/include/. /usr/include/
94+
95+
git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
96+
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
97+
-DgRPC_BUILD_TESTS=OFF \
98+
../.. && sudo make -j12 install
99+
90100
- name: Test
91101
run: |
92102
ESPEAK_DATA="/build/lib/Linux-$(uname -m)/piper_phonemize/lib/espeak-ng-data" GO_TAGS="tts stablediffusion" make test
@@ -108,6 +118,14 @@ jobs:
108118
# You can test your matrix by printing the current Go version
109119
- name: Display Go version
110120
run: go version
121+
- name: Dependencies
122+
run: |
123+
git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
124+
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
125+
-DgRPC_BUILD_TESTS=OFF \
126+
../.. && make -j12 install && rm -rf grpc
111127
- name: Test
112128
run: |
129+
export C_INCLUDE_PATH=/usr/local/include
130+
export CPLUS_INCLUDE_PATH=/usr/local/include
113131
CMAKE_ARGS="-DLLAMA_F16C=OFF -DLLAMA_AVX512=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF" make test

‎Dockerfile‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ ENV GALLERIES='[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/i
1616
ARG GO_TAGS="stablediffusion tts"
1717

1818
RUN apt-get update && \
19-
apt-get install -y ca-certificates cmake curl patch pip
19+
apt-get install -y ca-certificates curl patch pip cmake
20+
2021

2122
# Use the variables in subsequent instructions
2223
RUN echo "Target Architecture: $TARGETARCH"
@@ -104,6 +105,15 @@ RUN make prepare
104105
COPY . .
105106
COPY .git .
106107

108+
# stablediffusion does not tolerate a newer version of abseil, build it first
109+
RUN GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
110+
111+
RUN git clone --recurse-submodules -b v1.58.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
112+
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
113+
-DgRPC_BUILD_TESTS=OFF \
114+
../.. && make -j12 install && rm -rf grpc
115+
116+
# Rebuild with defaults backends
107117
RUN ESPEAK_DATA=/build/lib/Linux-$(uname -m)/piper_phonemize/lib/espeak-ng-data make build
108118

109119
###################################
@@ -132,8 +142,13 @@ WORKDIR /build
132142
# https://github.com/go-skynet/LocalAI/pull/434
133143
COPY . .
134144
RUN make prepare-sources
145+
146+
# Copy the binary
135147
COPY --from=builder /build/local-ai ./
136148

149+
# do not let piper rebuild (requires an older version of absl)
150+
COPY --from=builder /build/backend-assets/grpc/piper ./backend-assets/grpc/piper
151+
137152
# Copy VALLE-X as it's not a real "lib"
138153
RUN cp -rfv /usr/lib/vall-e-x/* ./
139154

‎Makefile‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ GOLLAMA_VERSION?=1676dcd7a139b6cdfbaea5fd67f46dc25d9d8bcf
88

99
GOLLAMA_STABLE_VERSION?=50cee7712066d9e38306eccadcfbb44ea87df4b7
1010

11+
CPPLLAMA_VERSION?=24ba3d829e31a6eda3fa1723f692608c2fa3adda
12+
1113
# gpt4all version
1214
GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
1315
GPT4ALL_VERSION?=27a8b020c36b0df8f8b82a252d261cda47cf44b8
@@ -120,7 +122,7 @@ ifeq ($(findstring tts,$(GO_TAGS)),tts)
120122
OPTIONAL_GRPC+=backend-assets/grpc/piper
121123
endif
122124

123-
GRPC_BACKENDS?=backend-assets/grpc/langchain-huggingface backend-assets/grpc/falcon-ggml backend-assets/grpc/bert-embeddings backend-assets/grpc/falcon backend-assets/grpc/bloomz backend-assets/grpc/llama backend-assets/grpc/llama-stable backend-assets/grpc/gpt4all backend-assets/grpc/dolly backend-assets/grpc/gpt2 backend-assets/grpc/gptj backend-assets/grpc/gptneox backend-assets/grpc/mpt backend-assets/grpc/replit backend-assets/grpc/starcoder backend-assets/grpc/rwkv backend-assets/grpc/whisper $(OPTIONAL_GRPC)
125+
GRPC_BACKENDS?=backend-assets/grpc/langchain-huggingface backend-assets/grpc/falcon-ggml backend-assets/grpc/bert-embeddings backend-assets/grpc/falcon backend-assets/grpc/bloomz backend-assets/grpc/llama backend-assets/grpc/llama-cpp backend-assets/grpc/llama-stable backend-assets/grpc/gpt4all backend-assets/grpc/dolly backend-assets/grpc/gpt2 backend-assets/grpc/gptj backend-assets/grpc/gptneox backend-assets/grpc/mpt backend-assets/grpc/replit backend-assets/grpc/starcoder backend-assets/grpc/rwkv backend-assets/grpc/whisper $(OPTIONAL_GRPC)
124126

125127
.PHONY: all test build vendor
126128

@@ -223,7 +225,7 @@ go-llama/libbinding.a: go-llama
223225
go-llama-stable/libbinding.a: go-llama-stable
224226
$(MAKE) -C go-llama-stable BUILD_TYPE=$(STABLE_BUILD_TYPE) libbinding.a
225227

226-
go-piper/libpiper_binding.a:
228+
go-piper/libpiper_binding.a: go-piper
227229
$(MAKE) -C go-piper libpiper_binding.a example/main
228230

229231
get-sources: go-llama go-llama-stable go-ggllm go-ggml-transformers gpt4all go-piper go-rwkv whisper.cpp go-bert bloomz go-stable-diffusion
@@ -280,6 +282,7 @@ clean: ## Remove build related file
280282
rm -rf ./go-ggllm
281283
rm -rf $(BINARY_NAME)
282284
rm -rf release/
285+
$(MAKE) -C backend/cpp/llama clean
283286

284287
## Build:
285288

@@ -395,6 +398,16 @@ ifeq ($(BUILD_TYPE),metal)
395398
cp go-llama/build/bin/ggml-metal.metal backend-assets/grpc/
396399
endif
397400

401+
backend/cpp/llama/grpc-server:
402+
LLAMA_VERSION=$(CPPLLAMA_VERSION) $(MAKE) -C backend/cpp/llama grpc-server
403+
404+
backend-assets/grpc/llama-cpp: backend-assets/grpc backend/cpp/llama/grpc-server
405+
cp -rfv backend/cpp/llama/grpc-server backend-assets/grpc/llama-cpp
406+
# TODO: every binary should have its own folder instead, so can have different metal implementations
407+
ifeq ($(BUILD_TYPE),metal)
408+
cp backend/cpp/llama/llama.cpp/build/bin/ggml-metal.metal backend-assets/grpc/
409+
endif
410+
398411
backend-assets/grpc/llama-stable: backend-assets/grpc go-llama-stable/libbinding.a
399412
$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama-stable
400413
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama-stable LIBRARY_PATH=$(shell pwd)/go-llama \
@@ -451,9 +464,12 @@ backend-assets/grpc/bert-embeddings: backend-assets/grpc go-bert/libgobert.a
451464
backend-assets/grpc/langchain-huggingface: backend-assets/grpc
452465
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./cmd/grpc/langchain-huggingface/
453466

454-
backend-assets/grpc/stablediffusion: backend-assets/grpc go-stable-diffusion/libstablediffusion.a
455-
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-stable-diffusion/ LIBRARY_PATH=$(shell pwd)/go-stable-diffusion/ \
456-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./cmd/grpc/stablediffusion/
467+
backend-assets/grpc/stablediffusion: backend-assets/grpc
468+
if [ ! -f backend-assets/grpc/stablediffusion ]; then \
469+
$(MAKE) go-stable-diffusion/libstablediffusion.a; \
470+
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-stable-diffusion/ LIBRARY_PATH=$(shell pwd)/go-stable-diffusion/ \
471+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./cmd/grpc/stablediffusion/; \
472+
fi
457473

458474
backend-assets/grpc/piper: backend-assets/grpc backend-assets/espeak-ng-data go-piper/libpiper_binding.a
459475
CGO_LDFLAGS="$(CGO_LDFLAGS)" LIBRARY_PATH=$(shell pwd)/go-piper \

‎backend/cpp/llama/CMakeLists.txt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
set(CMAKE_CXX_STANDARD 17)
2+
cmake_minimum_required(VERSION 3.15)
3+
set(TARGET grpc-server)
4+
set(_PROTOBUF_LIBPROTOBUF libprotobuf)
5+
set(_REFLECTION grpc++_reflection)
6+
7+
find_package(absl CONFIG REQUIRED)
8+
find_package(Protobuf CONFIG REQUIRED)
9+
find_package(gRPC CONFIG REQUIRED)
10+
11+
find_program(_PROTOBUF_PROTOC protoc)
12+
set(_GRPC_GRPCPP grpc++)
13+
find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
14+
15+
include_directories(${CMAKE_CURRENT_BINARY_DIR})
16+
include_directories(${Protobuf_INCLUDE_DIRS})
17+
18+
message(STATUS "Using protobuf ${Protobuf_VERSION} ${Protobuf_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}")
19+
20+
21+
# Proto file
22+
get_filename_component(hw_proto "../../../../../../pkg/grpc/proto/backend.proto" ABSOLUTE)
23+
get_filename_component(hw_proto_path "${hw_proto}" PATH)
24+
25+
# Generated sources
26+
set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.cc")
27+
set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.h")
28+
set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.cc")
29+
set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.h")
30+
31+
add_custom_command(
32+
OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
33+
COMMAND ${_PROTOBUF_PROTOC}
34+
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
35+
--cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
36+
-I "${hw_proto_path}"
37+
--plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
38+
"${hw_proto}"
39+
DEPENDS "${hw_proto}")
40+
41+
# hw_grpc_proto
42+
add_library(hw_grpc_proto
43+
${hw_grpc_srcs}
44+
${hw_grpc_hdrs}
45+
${hw_proto_srcs}
46+
${hw_proto_hdrs})
47+
48+
add_executable(${TARGET} grpc-server.cpp)
49+
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} absl::flags hw_grpc_proto
50+
absl::flags_parse
51+
gRPC::${_REFLECTION}
52+
gRPC::${_GRPC_GRPCPP}
53+
protobuf::${_PROTOBUF_LIBPROTOBUF})
54+
target_compile_features(${TARGET} PRIVATE cxx_std_11)
55+
if(TARGET BUILD_INFO)
56+
add_dependencies(${TARGET} BUILD_INFO)
57+
endif()

‎backend/cpp/llama/Makefile‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
2+
LLAMA_VERSION?=24ba3d829e31a6eda3fa1723f692608c2fa3adda
3+
4+
CMAKE_ARGS?=
5+
BUILD_TYPE?=
6+
7+
# If build type is cublas, then we set -DLLAMA_CUBLAS=ON to CMAKE_ARGS automatically
8+
ifeq ($(BUILD_TYPE),cublas)
9+
CMAKE_ARGS+=-DLLAMA_CUBLAS=ON
10+
# If build type is openblas then we set -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS
11+
# to CMAKE_ARGS automatically
12+
else ifeq ($(BUILD_TYPE),openblas)
13+
CMAKE_ARGS+=-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS
14+
# If build type is clblast (openCL) we set -DLLAMA_CLBLAST=ON -DCLBlast_DIR=/some/path
15+
else ifeq ($(BUILD_TYPE),clblast)
16+
CMAKE_ARGS+=-DLLAMA_CLBLAST=ON -DCLBlast_DIR=/some/path
17+
# If it's hipblas we do have also to set CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++
18+
else ifeq ($(BUILD_TYPE),hipblas)
19+
CMAKE_ARGS+=-DLLAMA_HIPBLAS=ON
20+
endif
21+
22+
llama.cpp:
23+
git clone --recurse-submodules https://github.com/ggerganov/llama.cpp llama.cpp
24+
cd llama.cpp && git checkout -b build $(LLAMA_VERSION) && git submodule update --init --recursive --depth 1
25+
26+
llama.cpp/examples/grpc-server:
27+
mkdir -p llama.cpp/examples/grpc-server
28+
cp -r $(abspath ./)/CMakeLists.txt llama.cpp/examples/grpc-server/
29+
cp -r $(abspath ./)/grpc-server.cpp llama.cpp/examples/grpc-server/
30+
echo "add_subdirectory(grpc-server)" >> llama.cpp/examples/CMakeLists.txt
31+
32+
rebuild:
33+
cp -rfv $(abspath ./)/CMakeLists.txt llama.cpp/examples/grpc-server/
34+
cp -rfv $(abspath ./)/grpc-server.cpp llama.cpp/examples/grpc-server/
35+
rm -rf grpc-server
36+
$(MAKE) grpc-server
37+
38+
clean:
39+
rm -rf llama.cpp
40+
rm -rf grpc-server
41+
42+
grpc-server: llama.cpp llama.cpp/examples/grpc-server
43+
cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release
44+
cp llama.cpp/build/bin/grpc-server .

0 commit comments

Comments
 (0)