CMake : cross-compilation

MacOSX and iOS
1

Im using a mac mini M1 (apple silicon) using CMake to compile VSTs

works for native compile on mac, my same compile also works on a Linux machine.

however, Im trying to use make to cross compile on the M1 to armhf.
my cross compilation environment is working fine for my other cmake projects.
but not for juce cmake

what appears to be happening is when we build the cmake file juce cmake is bringing in dependanies from the local environment rather than sysroot

e.g

bash-3.2$ cmake -DCMAKE_TOOLCHAIN_FILE=../xc.cmake ..
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
-- The C compiler identification is Clang 11.0.0
-- The CXX compiler identification is Clang 11.0.0
-- Detecting C compiler ABI info
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /opt/homebrew/opt/llvm/bin/clang - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /opt/homebrew/opt/llvm/bin/clang++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
-- Found PkgConfig: /opt/homebrew/bin/pkg-config (found version "0.29.2") 
-- Checking for module 'libcurl'
--   No package 'libcurl' found
-- Checking for modules 'webkit2gtk-4.0;gtk+-x11-3.0'
--   No package 'webkit2gtk-4.0' found
--   No package 'gtk+-x11-3.0' found
using xc-ssp-toolchain
defaulting XC_ROOT to ~/xc/xcSSP
using envvar XC_ROOT  - /Users/markh/xc/xcSSP
-- Checking for module 'alsa'
--   No package 'alsa' found
-- Checking for module 'freetype2'
--   Found freetype2, version 23.4.17
-- Configuring juceaide
-- Building juceaide


-- Exporting juceaide
-- Configuring done
-- Generating done
-- Build files have been written to: /Users/markh/projects/SSP.private/build.xc

here we can see its not finding a bunch of libraries (webkit2gtk-4.0) since they are not in the local environment (only the sysroot) , and similar freetype is the wrong verison 23.4.17 is the local env, where sysroot has 18.3.12

we can see this materialise when we try to link

cd /Users/markh/projects/SSP.private/build.xc/technobear/clds && /opt/homebrew/Cellar/cmake/3.19.1/bin/cmake -E cmake_link_script CMakeFiles/CLDS_VST.dir/link.txt --verbose=1
/opt/homebrew/opt/llvm/bin/clang++ --target=arm-linux-gnueabihf --sysroot=/Users/markh/xc/xcSSP/sysroot -fPIC  -mcpu=cortex-a17 -mfloat-abi=hard -mfpu=neon-vfpv4 -mcpu=cortex-a17 -mfloat-abi=hard -mfpu=neon-vfpv4   -L/Users/markh/xc/xcSSP/sysroot/usr/local/lib -L/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf  -B/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf -Wl,-rpath-link,/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf  -L/Users/markh/xc/xcSSP/sysroot/usr/local/lib -L/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf  -B/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf -Wl,-rpath-link,/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/local/lib -L/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf  -B/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf -Wl,-rpath-link,/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -shared  -o CLDS_artefacts/VST/clds.so CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_AAX.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_1.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_2.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_3.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_4.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_utils.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_Standalone.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_Unity.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_VST2.cpp.o CMakeFiles/CLDS_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_VST3.cpp.o  -Wl,--no-undefined CLDS_artefacts/libclds_SharedCode.a /opt/homebrew/opt/freetype/lib/libfreetype.a -lrt -ldl -lpthread 
/opt/homebrew/bin/arm-linux-gnueabihf-ld: CLDS_artefacts/libclds_SharedCode.a(juce_gui_basics.cpp.o): in function `juce::XWindowSystem::setScreenSaverEnabled(bool) const':

note: how its used /opt/homebrew/opt/freetype/lib/libfreetype.a

whereas it should be using /Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf/libfreetype.a
( sysroot = /Users/markh/xc/xcSSP/sysroot)

obviously these leads to a whole bunch of link errors since the compilation has correctly been using sysroot.

any thoughts where to check in the juce cmake configuration?

to reiterateā€¦ this cross compile environment and cmake toolchain file works fine for other cmake projects I have.

for clarity, Im using cmake toolchains, as discussed here:

Im then using arm-linux-gnueabihf for the cross compilation using clang

2

Cross-compilation of JUCE with CMake is not ā€œofficiallyā€ supported and has not been tested. It may be possible to get it working, though.

My best guess is that pkg-config is failing to search the correct locations for the dependencies. This post on stackoverflow looks like it might contain a potential fix.

1 Like
3

thanks @reuk that was really useful :slight_smile:

ok, so Im very closeā€¦ but getting a couple of wierd link errors

[100%] Linking CXX shared module MSW8_artefacts/VST/msw8.so
cd /Users/markh/projects/SSP.private/build.xc/technobear/msw8 && /opt/homebrew/Cellar/cmake/3.19.1/bin/cmake -E cmake_link_script CMakeFiles/MSW8_VST.dir/link.txt --verbose=1
/opt/homebrew/opt/llvm/bin/clang++ --target=arm-linux-gnueabihf --sysroot=/Users/markh/xc/xcSSP/sysroot -fPIC  -mcpu=cortex-a17 -mfloat-abi=hard -mfpu=neon-vfpv4  	 -L/Users/markh/xc/xcSSP/sysroot/lib/gcc/arm-linux-gnueabihf  -B/Users/markh/xc/xcSSP/sysroot/lib/gcc/arm-linux-gnueabihf -Wl,-rpath-link,/Users/markh/xc/xcSSP/sysroot/usr/lib/arm-linux-gnueabihf -L/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf  -B/Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf -Wl,-rpath-link,/Users/markh/xc/xcSSP/sysroot/lib/arm-linux-gnueabihf -shared  -o MSW8_artefacts/VST/msw8.so CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_AAX.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_1.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_2.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_3.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_4.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_RTAS_utils.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_Standalone.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_Unity.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_VST2.cpp.o CMakeFiles/MSW8_VST.dir/__/__/juce/modules/juce_audio_plugin_client/juce_audio_plugin_client_VST3.cpp.o  -Wl,--no-undefined MSW8_artefacts/libmsw8_SharedCode.a -lfreetype -lasound -lrt -ldl -lpthread 
/opt/homebrew/bin/arm-linux-gnueabihf-ld: MSW8_artefacts/libmsw8_SharedCode.a(juce_gui_basics.cpp.o): in function `juce::XWindowSystem::setScreenSaverEnabled(bool) const':
juce_gui_basics.cpp:(.text+0xabc3c): warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libm.a(s_atan.o): relocation R_ARM_THM_MOVW_ABS_NC against `a local symbol' can not be used when making a shared object; recompile with -fPIC
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libm.a(halfulp.o): relocation R_ARM_THM_MOVW_ABS_NC against `a local symbol' can not be used when making a shared object; recompile with -fPIC
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libm.a(mpexp.o): relocation R_ARM_THM_MOVW_ABS_NC against `a local symbol' can not be used when making a shared object; recompile with -fPIC

.... lots of similar issues, all surround libm...

/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libdl.a(dlopen.o): in function `dlopen':
(.text+0x8): undefined reference to `__dlopen'
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libdl.a(dlclose.o): in function `dlclose':
(.text+0x0): undefined reference to `__dlclose'
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libdl.a(dlsym.o): in function `dlsym':
(.text+0x8): undefined reference to `__dlsym'
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libdl.a(dladdr.o): in function `dladdr':
(.text+0x0): undefined reference to `__dladdr'
/opt/homebrew/bin/arm-linux-gnueabihf-ld: /Users/markh/xc/xcSSP/sysroot/usr/lib/gcc/arm-linux-gnueabihf/6.3.0/../../../arm-linux-gnueabihf/libm.a(k_standard.o)(.text+0x46e): unresolvable R_ARM_THM_MOVW_ABS_NC relocation against symbol `stderr@@GLIBC_2.4'
clang-11: error: linker command failed with exit code 1 (use -v to see invocation)
make[3]: *** [technobear/msw8/MSW8_artefacts/VST/msw8.so] Error 1
make[2]: *** [technobear/msw8/CMakeFiles/MSW8_VST.dir/all] Error 2
make[1]: *** [technobear/msw8/CMakeFiles/MSW8_VST.dir/rule] Error 2

issue one isā€¦
relocation R_ARM_THM_MOVW_ABS_NC against a local symbolā€™ can not be used when making a shared object; recompile with -fPIC`

Ive checked everything is being compiled with -fPIC already !!!
it looks like something is pulling in libm statically, but it is available as a dynamic lib in sysroot

./usr/lib/arm-linux-gnueabihf/libm.a
./lib/arm-linux-gnueabihf/libm.so.6
./lib/arm-linux-gnueabihf/libm-2.24.so

Im similarly confused why __dlopen etc is missing, presumably this is from dlopen() which is being resolved via -ldl

finally get anotehr unresolved error for stderrā€¦ from glibc
unresolvable R_ARM_THM_MOVW_ABS_NC relocation against symbol stderr@@GLIBC_2.4ā€™`

so, not quite sure what Im missing or what is causing the issuesā€¦
the compile and links are all referring correctly now to the sysroot, so I dont think is contamination from the native environment.

thoughts?

4

According to this answer (thanks stackoverflow!), it could be that the linker isnā€™t finding the shared libraries in /lib/arm-linux-gnueabihf, and is looking in /usr/lib/... instead, where it only finds the staticlib.

1 Like
5

thanks againā€¦
so, yeah, the 2nd answer is actually the correct one

the issue in my case (and I suspect for the OP @ stackoverflow) was caused when creating sysroot.
basically, I rsync the sysroot from a ā€˜donorā€™ machine, the issue is rsync was not copying across if symlinks had absolute pathsā€¦

SO /usr/lib/arm-linux-gnueabihf/libm.so ā†’ /lib/arm-linux-gnueabihf/libm.so.6 was not copying libm.so overā€¦ so as pointed out, libm.so was missing, so it reverted to libm.a (!)
(the absolute path caught me out, since a most libs with relative paths were fine)

so upshot is, I need to review my rsync options for creating the sysroot.

for now, sorted out libm/libdl out - and ā€¦

hey presto it all works !!!

cross compiling on cmake for an armhf platform.
and wow, its SO quickā€¦

8 VSTs from a fresh build, compiled/linked in (just) less than 1 minute!

thanks for your help
Mark

in case its useful anyone doing this, here is my working cross-compile toolchain file
youā€™ll need to adapt for your project/environment, but hopefully gives a starting point.
(XC ROOT = path to my cross compile environment)

if(DEFINED ENV{XC_ROOT})
    set(XC_ROOT $ENV{XC_ROOT})
    message("using envvar XC_ROOT  - ${XC_ROOT}")
else()
    message("defaulting XC_ROOT to ~/xc/xcSSP")
    get_filename_component(XC_ROOT "~/xc/xcSSP" ABSOLUTE)
    message("using envvar XC_ROOT  - ${XC_ROOT}")
endif()


set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)
set(triple arm-linux-gnueabihf)

set(CMAKE_C_COMPILER /opt/homebrew/opt/llvm/bin/clang)
set(CMAKE_CXX_COMPILER /opt/homebrew/opt/llvm/bin/clang++)
# set(tools /opt/homebrew/opt/arm-linux-gnueabihf-binutils)
set(CMAKE_C_COMPILER_TARGET ${triple})
set(CMAKE_CXX_COMPILER_TARGET ${triple})

set(CMAKE_SYSROOT ${XC_ROOT}/sysroot)

set(ENV{PKG_CONFIG_DIR} "")
set(ENV{PKG_CONFIG_LIBDIR} "${CMAKE_SYSROOT}/usr/lib/arm-linux-gnueabihf/pkgconfig:${CMAKE_SYSROOT}/usr/share/pkgconfig")
set(ENV{PKG_CONFIG_SYSROOT_DIR} ${CMAKE_SYSROOT})


set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -L${CMAKE_SYSROOT}/lib -B${CMAKE_SYSROOT}/lib")
set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -Wl,-rpath-link,${CMAKE_SYSROOT}/lib")

set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -L${CMAKE_SYSROOT}/usr/lib/arm-linux-gnueabihf -B${CMAKE_SYSROOT}/usr/lib/arm-linux-gnueabihf ")
set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -Wl,-rpath-link,${CMAKE_SYSROOT}/usr/lib/arm-linux-gnueabihf")

set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -L${CMAKE_SYSROOT}/lib/arm-linux-gnueabihf  -B${CMAKE_SYSROOT}/lib/arm-linux-gnueabihf")
set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -Wl,-rpath-link,${CMAKE_SYSROOT}/lib/arm-linux-gnueabihf")

set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -L${CMAKE_SYSROOT}/usr/lib/gcc/arm-linux-gnueabihf  		 -B${CMAKE_SYSROOT}/usr/lib/gcc/arm-linux-gnueabihf")
set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -L${CMAKE_SYSROOT}/usr/lib/gcc/arm-linux-gnueabihf/6.3.0   -B${CMAKE_SYSROOT}/usr/lib/gcc/arm-linux-gnueabihf/6.3.0 ")
set(SSP_LINK_FLAGS "${SSP_LINK_FLAGS} -Wl,-rpath-link,${CMAKE_SYSROOT}/usr/lib/gcc/arm-linux-gnueabihf/6.3.0")



set(CMAKE_EXE_LINKER_FLAGS 		"${CMAKE_EXE_LINKER_FLAGS} 		${SSP_LINK_FLAGS}")
set(CMAKE_MODULE_LINKER_FLAGS 	"${CMAKE_MODULE_LINKER_FLAGS} 	${SSP_LINK_FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS 	"${CMAKE_SHARED_LINKER_FLAGS} 	${SSP_LINK_FLAGS}")

set(CMAKE_C_FLAGS 	"${CMAKE_C_FLAGS} -mcpu=cortex-a17 -mfloat-abi=hard -mfpu=neon-vfpv4")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=cortex-a17 -mfloat-abi=hard -mfpu=neon-vfpv4")
6

Nice, glad you got it working!