A Mutable Log

Cross-compile gRPC on Ubuntu 20.04

This post provides instructions on building gRPC v1.8.5 for ARM using Clang++ on a Ubuntu 20.04 host.

Install build dependencies

sudo apt install autoconf automake libtool build-essential curl unzip pkg-config libasound2-dev libcurl4-openssl-dev

Build and install gRPC on host

The same version of gRPC needs to be built and installed on the host first when cross-compiling.

sudo apt-get purge libc-ares-dev  # https://github.com/grpc/grpc/pull/10706#issuecomment-302775038
sudo apt-get purge libprotobuf-dev libprotoc-dev
sudo rm -rf /usr/local/bin/grpc_* /usr/local/bin/protoc \
    /usr/local/include/google/protobuf/ /usr/local/include/grpc/ /usr/local/include/grpc++/ \
    /usr/local/lib/libproto* /usr/local/lib/libgpr* /usr/local/lib/libgrpc* \
    /usr/local/lib/pkgconfig/protobuf* /usr/local/lib/pkgconfig/grpc* \
    /usr/local/share/grpc/

Clone repo

git clone -b v1.8.5 https://github.com/grpc/grpc
cd grpc

Patch Makefile

diff --git a/Makefile b/Makefile
index 0f2ccb6ea7..954ccbb578 100644
--- a/Makefile
+++ b/Makefile
@@ -327,7 +327,7 @@ CXXFLAGS += -std=c++11
 ifeq ($(SYSTEM),Darwin)
 CXXFLAGS += -stdlib=libc++
 endif
-CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
+CPPFLAGS += -g -Wall -Wextra -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
 COREFLAGS += -fno-rtti -fno-exceptions
 LDFLAGS += -g

Patch src/core/lib/support/log_linux.cc

diff --git a/src/core/lib/support/log_linux.cc b/src/core/lib/support/log_linux.cc
index e0e277fe87..c70c187fb3 100644
--- a/src/core/lib/support/log_linux.cc
+++ b/src/core/lib/support/log_linux.cc
@@ -39,7 +39,7 @@
 #include <time.h>
 #include <unistd.h>
 
-static long gettid(void) { return syscall(__NR_gettid); }
+static long local_gettid(void) { return syscall(__NR_gettid); }
 
 void gpr_log(const char* file, int line, gpr_log_severity severity,
              const char* format, ...) {
@@ -65,7 +65,7 @@ extern "C" void gpr_default_log(gpr_log_func_args* args) {
   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
   struct tm tm;
   static __thread long tid = 0;
-  if (tid == 0) tid = gettid();
+  if (tid == 0) tid = local_gettid();
 
   timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '/');

Continue build

git submodule update --init

cd third_party/protobuf
./autogen.sh && ./configure && make
sudo make install
sudo ldconfig

cd ../..
make clean
make
sudo make install
sudo ldconfig

Build and install gRPC to target SDK location

In the code snippets below $SDKTARGETSYSROOT should point to the cross-compile toolchain sysroot.

Remove previous build

sudo rm -rf $SDKTARGETSYSROOT/usr/local/bin/grpc_* $SDKTARGETSYSROOT/usr/local/bin/protoc \
    $SDKTARGETSYSROOT/usr/local/include/google/protobuf/ $SDKTARGETSYSROOT/usr/local/include/grpc/ $SDKTARGETSYSROOT/usr/local/include/grpc++/ \
    $SDKTARGETSYSROOT/usr/local/lib/libproto* $SDKTARGETSYSROOT/usr/local/lib/libgpr* $SDKTARGETSYSROOT/usr/local/lib/libgrpc* \
    $SDKTARGETSYSROOT/usr/local/lib/pkgconfig/protobuf* $SDKTARGETSYSROOT/usr/local/lib/pkgconfig/grpc* \
    $SDKTARGETSYSROOT/usr/local/share/grpc/

Clone repo again—I recommend not using the location used to build for host

git clone -b v1.8.5 https://github.com/grpc/grpc

Patch Makefile

diff --git a/Makefile b/Makefile
index 0f2ccb6ea7..954ccbb578 100644
--- a/Makefile
+++ b/Makefile
@@ -327,7 +327,7 @@ CXXFLAGS += -std=c++11
 ifeq ($(SYSTEM),Darwin)
 CXXFLAGS += -stdlib=libc++
 endif
-CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
+CPPFLAGS += -g -Wall -Wextra -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1
 COREFLAGS += -fno-rtti -fno-exceptions
 LDFLAGS += -g
 
@@ -796,6 +796,9 @@ else
 PC_LIBS_GRPCXX = -lprotobuf
 endif
 PROTOC_PLUGINS = $(PROTOC_PLUGINS_ALL)
+ifeq ($(USE_BUILT_PROTOC),false)
+PROTOC_PLUGINS_DIR = $(prefix)/bin
+endif
 else
 ifeq ($(HAS_EMBEDDED_PROTOBUF),true)
 PROTOBUF_DEP = $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a

Continue build

git submodule update --init

cd third_party/protobuf

# This works around a libtool bug:
# https://debbugs.gnu.org/cgi/bugreport.cgi?bug=27866.
#
# Change usr/lib/clang/8.0.1/lib/linux and clang_rt.builtins-armhf
# to something appropriate for your environment.
export CXXFLAGS=" -O2 -pipe -g -feliminate-unused-debug-types -Wno-error=unused-command-line-argument -Qunused-arguments --rtlib=compiler-rt -stdlib=libc++ -L$SDKTARGETSYSROOT/usr/lib/clang/8.0.1/lib/linux -lclang_rt.builtins-armhf "

# Run git clean -fdX before building again
./autogen.sh && ./configure --host=arm-linux --with-protoc=/usr/local/bin/protoc && make
sudo "PATH=$PATH" -E make DESTDIR=$SDKTARGETSYSROOT install

cd ../..
export GRPC_CROSS_COMPILE=true
export GRPC_CROSS_AROPTS="rc --target=elf32-littlearm"
export PROTOBUF_CONFIG_OPTS="--host=arm-linux --with-protoc=/usr/local/bin/protoc"
export PKG_CONFIG_PATH="$SDKTARGETSYSROOT/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH"
export LD="$CC $CFLAGS"
export LDXX="$CXX $CXXFLAGS"
make clean
make
sudo "PATH=$PATH" -E make prefix=$SDKTARGETSYSROOT/usr/local install

Use patchelf --print-needed to determine dependencies needed to run apps. Additionally, gRPC requires /usr/local/share/grpc/roots.pem or you’ll get the following error at runtime

E1123 15:32:02.270457781    1018 security_connector.cc:967]  load_file: {"created":"@1606174322.270361781","description":"Failed to load file","file":"src/core/lib/iomgr/load_file.cc","file_line":69,"filename":"/usr/local/share/grpc/roots.pem","referenced_errors":[{"created":"@1606174322.270345242","description":"OS Error","errno":2,"file":"src/core/lib/iomgr/load_file.cc","file_line":43,"os_error":"No such file or directory","syscall":"fopen"}]}
E1123 15:32:02.270812781    1018 security_connector.cc:1013] Could not get default pem root certs.

Alternatively, you can point to root certificates on Debian and derivatives, as follows, before you run your app

export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt

At this point, you should be able to compile and build apps that require gRPC.