Last active
October 24, 2023 21:39
-
-
Save ChadJPetersen/e5215da9c00b20fabff05b4a0e43e385 to your computer and use it in GitHub Desktop.
After hours of slaving I finally got Xapian working on windows with a ruby binding (I use it for Redmine). I know this is far from the right way to do it, and is rather hacky. But it works! Also I found little starting points online when trying to compile Xapian on windows. So hoping this helps someone else a bit!
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Install MSYS2 https://www.msys2.org/ | |
| # In MSYS2 console | |
| pacman -Syu | |
| # close and reopen | |
| pacman -Su | |
| # Close and reopen | |
| # if getting errors on the below you can try the --force option. | |
| # The commented out i686 versions of pacman packages are for a 32 bit build. | |
| pacman -Syuu | |
| pacman -S base-devel --needed | |
| pacman -S mingw-w64-x86_64-toolchain | |
| # pacman -S mingw-w64-i686-toolchain | |
| pacman -S zlib | |
| pacman -S file | |
| pacman -S pcre | |
| pacman -S mingw64/mingw-w64-x86_64-file | |
| pacman -S mingw64/mingw-w64-x86_64-pcre | |
| pacman -R libtool | |
| pacman -S mingw64/mingw-w64-x86_64-libtool | |
| # pacman -S mingw64/mingw-w64-i686-libtool | |
| # pacman -S mingw32/mingw-w64-i686-file | |
| # pacman -S mingw32/mingw-w64-i686-pcre | |
| # pacman -S mingw-w64-i686-cmake | |
| pacman -S mingw-w64-x86_64-cmake | |
| pacman -S gcc | |
| pacman -S mingw64/mingw-w64-x86_64-zlib | |
| pacman -S msys/zlib-devel | |
| # pacman -S mingw32/mingw-w64-i686-zlib | |
| # In MINGW64 or 32 Console depending on your need. | |
| # BE SURE TO UPDATE PATH'S TO WHERE YOU WANT THEM!!! | |
| cd /C/Tools/xapian-core-1.4.7 | |
| ./configure --prefix=/C/Tools/Xapian --enable-shared --disable-static LDFLAGS="-Wl,--no-undefined -Wl,-ldxerr9 -L/C/Tools/msys64/mingw64/lib" CFLAGS=-fPIC CXXFLAGS=-fPIC | |
| #patch lib tool using this bash function | |
| libtool_patch() | |
| { | |
| echo "Applying patch for libtool pe-x86_64 magic.." | |
| rm -f tmp.0lt.diff && touch tmp.0lt.diff | |
| lst="libtool.m4 aclocal.m4 ltmain.sh libtool configure" | |
| for i in $lst | |
| do | |
| j=`find . -type f -name $i` | |
| for f in $j | |
| do | |
| if test -f $f; then | |
| cp -p $f tmp.0lt | |
| sed -e "s|EGREP[-e ]*'file format pe-i386(\.\*architecture: i386)?' >|EGREP 'file format (pei\*-i386(.\*architecture:\ i386)?\|pe-arm-wince\|pe-x86-64)' >|" \ | |
| -e "s|EGREP[-e ]*'file format (pe-i386(\.\*architecture: i386)?\|pe-arm-wince\|pe-x86-64)' >|EGREP 'file format (pei\*-i386(.\*architecture:\ i386)?\|pe-arm-wince\|pe-x86-64)' >|" \ | |
| -e "s|deplibs_check_method='file_magic file format pei\*-i386(\.\*architecture: i386)?'|deplibs_check_method='file_magic file format (pei\*-i386(.\*architecture:\ i386)?\|pe-arm-wince\|pe-x86-64)'|" \ | |
| -e "s|deplibs_check_method='file_magic file format pe-i386(\.\*architecture: i386)?'|deplibs_check_method='file_magic file format (pei\*-i386(.\*architecture:\ i386)?\|pe-arm-wince\|pe-x86-64)'|" \ | |
| tmp.0lt > $f | |
| diff -up tmp.0lt $f >> tmp.0lt.diff | |
| # preserve the original file timestamp to avoid autotools' reconfiguration | |
| touch -r tmp.0lt $f | |
| rm -f tmp.0lt | |
| fi | |
| done | |
| done | |
| diffstat -l -p0 tmp.0lt.diff 2> /dev/null | |
| } | |
| libtool_patch | |
| make | |
| # If you run into an issue about linkers not being able to find dynamic libraries and only a static library will be created then use this addition to the libtools file | |
| # But libtool could also be installed somewhere else on the path. Just be sure to grab the one that is being run by make. | |
| # Added lines with + | |
| # *executable*) # but shell scripts are "executable" too... | |
| # case $win32_fileres in | |
| # *MS\ Windows\ PE\ Intel*) | |
| # win32_libid_type="x86 DLL" | |
| # ;; | |
| # + *MS-DOS\ executable*) # ugh! some DLLs built on Windows | |
| # + win32_libid_type="x86 DLL" | |
| # + ;; | |
| # esac | |
| # The path to this file is C:\Tools\xapian-core-1.4.7\libtool | |
| # about line 5487 | |
| # The other option to get arround the linker warning to output a dynamic library instead of a static library is to recompile the libraries on x86 windows instead of x86 dos. | |
| # More info can be found here. http://ricardo.ecn.wfu.edu/~cottrell/cross-gtk/libtool.html | |
| # There are also notes that msys may not like being any place other than c:/msys64 So you may also need to create a symlink if things do not work. | |
| # (admin powershell) | |
| # cd C:\ | |
| # new-item -itemtype symboliclink -name msys64 -target C:\Tools\msys64\ | |
| # It may also be a missmatch between linux and windows naming. If it's failing on any file starting with lib search the windows folder for everything after the lib. | |
| # e.g. if failing on libpcrt4 copy pcrt4.dll out of windows directory into lib directory and rename it libprct4.dll | |
| # I also had to download a different binary from the zlib site here. http://www.winimage.com/zLibDll/ | |
| # If all else fails when trying to compile. Look at where lib tool is finding canidates and edit the make file after the configure step to point directly at the dll names (e.g. libz.dll instead of -lz) | |
| # If you just cannot get the warning about static libraries to go away... sad but not the end of the world. You can just follow the extended instructions at the end of this file. | |
| # run as admin | |
| make install | |
| # back in normal mode | |
| cd /C/Tools/xapian-omega-1.4.7 | |
| ./configure --prefix=/C/Tools/Xapian XAPIAN_CONFIG=/C/Tools/Xapian/bin/xapian-config --enable-shared --disable-static LDFLAGS="-Wl,--no-undefined -L/C/Tools/msys64/mingw64/lib" CFLAGS=-fPIC CXXFLAGS=-fPIC | |
| # You may need to add a file for mingw for the time strip function which can be downloaded here | |
| # https://raw.githubusercontent.com/Alexpux/MINGW-packages/master/mingw-w64-libkml/strptime.c | |
| make | |
| # run as admin | |
| make install | |
| # back in normal mode | |
| cd /C/Tools/xapian-bindings-1.4.7 | |
| # I have found that not all of the documentation of required parameters was listed in the --help section. It may be helpful to just read the configure file if you're having troubles. | |
| ./configure --prefix=/C/Tools/Xapian XAPIAN_CONFIG=/C/Tools/Xapian/bin/xapian-config --with-ruby RUBY="/C/Ruby24-x64/bin/ruby.exe" RDOC="/C/Ruby24-x64/bin/rdoc.cmd" RUBY_INC="/C/Ruby24-x64/include/ruby-2.4.0" RUBY_INC_ARCH="/C/Ruby24-x64/include/ruby-2.4.0/x64-mingw32" --disable-static --enable-shared LDFLAGS="-Wl,--no-undefined -L/C/Tools/msys64/mingw64/lib" CFLAGS=-fPIC CXXFLAGS=-fPIC | |
| make | |
| # Run as admin | |
| make install | |
| # If you cannot get xapian to build as a shared object follow these steps. | |
| 1.) Create a new directory structure that looks like this. | |
| CustomRubyGemDirectoryRoot | |
| │ Rakefile (see content below) | |
| │ xapian.gemspec (see content below) | |
| │ | |
| ├───ext | |
| │ ├───lib | |
| │ │ libxapian.a (this file is coppied from C:\Tools\Xapian\lib) | |
| │ └───_xapian | |
| │ │ extconf.rb (see content below) | |
| │ │ xapian.h (This file can be found here C:\Tools\Xapian\include\xapian.h) | |
| │ │ _xapian.cc (this file is coppied from this file and renamed C:\Tools\xapian-bindings-1.4.7\ruby\xapian_wrap.cc) | |
| │ │ _xapian.h (this file is coppied from this file and renamed C:\Tools\xapian-bindings-1.4.7\ruby\xapian_wrap.h) | |
| │ │ | |
| │ ├───lib | |
| │ │ libxapian.a (this file is coppied from C:\Tools\Xapian\lib) | |
| │ │ | |
| │ └───xapian (This folder can be found hereC:\Tools\Xapian\include\xapian) | |
| │ attributes.h | |
| │ compactor.h | |
| │ constants.h | |
| │ constinfo.h | |
| │ database.h | |
| │ dbfactory.h | |
| │ deprecated.h | |
| │ derefwrapper.h | |
| │ document.h | |
| │ enquire.h | |
| │ error.h | |
| │ errorhandler.h | |
| │ eset.h | |
| │ expanddecider.h | |
| │ geospatial.h | |
| │ intrusive_ptr.h | |
| │ iterator.h | |
| │ keymaker.h | |
| │ matchspy.h | |
| │ mset.h | |
| │ positioniterator.h | |
| │ postingiterator.h | |
| │ postingsource.h | |
| │ query.h | |
| │ queryparser.h | |
| │ registry.h | |
| │ stem.h | |
| │ termgenerator.h | |
| │ termiterator.h | |
| │ types.h | |
| │ unicode.h | |
| │ valueiterator.h | |
| │ valuesetmatchdecider.h | |
| │ version.h | |
| │ visibility.h | |
| │ weight.h | |
| │ | |
| └───lib | |
| xapian.rb (coppied from C:\Tools\xapian-bindings-1.4.7\ruby\xapian.rb) | |
| #end tree | |
| # Rakefile | |
| require 'rake/extensiontask' | |
| spec = Gem::Specification.load('xapian.gemspec') | |
| Rake::ExtensionTask.new do |ext| | |
| ext.name = '_xapian' # indicate the name of the extension. | |
| ext.ext_dir = 'ext/_xapian' # search for 'hello_world' inside it. | |
| # ext.lib_dir = 'lib/my_lib' # put binaries into this folder. | |
| # ext.config_script = 'extconf.rb' # use instead of the default 'extconf.rb'. | |
| ext.tmp_dir = 'tmp' # temporary folder used during compilation. | |
| ext.source_pattern = "*.{c,cpp,cc}" # monitor file changes to allow simple rebuild. | |
| # ext.config_options << 'CPPFLAGS="-I/C/Tools/Xapian/include"' # supply additional options to configure script. | |
| ext.gem_spec = spec # optionally indicate which gem specification will be used. | |
| end | |
| #end rakefile | |
| # xapian.gemspec | |
| Gem::Specification.new do |s| | |
| s.name = "_xapian" | |
| s.version = "1.4.7" | |
| s.summary = "Xapian for Ruby" | |
| s.author = "Edited by Chad Petersen" | |
| s.files = Dir.glob("ext/**/*.{c,rb}") + | |
| Dir.glob("lib/**/*.rb") | |
| s.extensions << "ext/_xapian/extconf.rb" | |
| s.add_development_dependency "rake-compiler" | |
| end | |
| #end xapian.gemspec | |
| # extconf.rb | |
| require "mkmf" | |
| LIBDIR = RbConfig::CONFIG['libdir'] | |
| INCLUDEDIR = RbConfig::CONFIG['includedir'] | |
| HEADER_DIRS = [INCLUDEDIR] | |
| # setup constant that is equal to that of the file path that holds that static libraries that will need to be compiled against | |
| LIB_DIRS = [LIBDIR, File.expand_path(File.join(File.dirname(__FILE__), "lib"))] | |
| # array of all libraries that the C extension should be compiled against | |
| libs = ['-lxapian','-lz','-lrpcrt4','-lws2_32'] | |
| dir_config('_xapian', HEADER_DIRS, LIB_DIRS) | |
| # iterate though the libs array, and append them to the $LOCAL_LIBS array used for the makefile creation | |
| libs.each do |lib| | |
| $LOCAL_LIBS << "#{lib} " | |
| end | |
| create_makefile('_xapian/_xapian') | |
| # end extconf.rb | |
| # Continue instruction steps. | |
| 2.) in powershell gem.cmd install rake-compiler | |
| 3.) cd to the root of the directory made above (CustomRubyGemDirectoryRoot). | |
| 4.) run 'rake compile' without the quotes. | |
| 5.) get the _xapian.so file that was generated and the xapian.rb file and put them into the ruby directory here C:\Ruby24-x64\lib\ruby\site_ruby\2.4.0 where 2.4.0 is the version number. | |
| 6.) Enjoy xapian |
Author
I haven't gotten the DLL to build. I found a way to get the so file that the ruby gem wants and stopped there.
I've updated the script a bit to include install of some dependencies that have since come up.
If you do get the DLL building I'd love a link here to your script though!
I found a way to get the DLL to build.
Its:
C:\tools\xapian-bindings\csharp>g++ -shared -o xapian.dll xapian_wrap.o C:\tools\xapian\lib\libxapian.a C:\tools\xapian\lib\libgetopt.a C:\msys64\mingw64\x86_64-w64-mingw32\lib\libwsock32.a C:\msys64\mingw64\x86_64-w64-mingw32\lib\libmincore.a C:\msys64\mingw64\lib\libz.a
My bash script for Xapian-Core:
./configure --prefix=/C/Tools/Xapian --enable-shared --disable-static LDFLAGS="-Wl,--no-undefined -Wl,-ldxerr9 -L/C/Tools/msys64/mingw64/lib" CFLAGS=-fPIC CXXFLAGS=-fPIC
My bash script for Xapian-Language-Integration:
./configure --prefix=/C/Tools/Xapian --enable-shared --disable-static LDFLAGS="-Wl,--no-undefined -Wl,-ldxerr9 -L/C/Tools/msys64/mingw64/lib" CFLAGS=-fPIC CXXFLAGS=-fPIC XAPIAN_CONFIG=/c/tools/xapian/xapian-config --with-csharp CSC=/c/progra~2/micros~1/2019/Community/MsBuild/Current/bin/Roslyn/csc.exe GACUTIL=/c/progra~2/micros~1/windows/v10.0a/bin/NETFX4~1.8TO/gacutil.exe GACUTIL=/c/progra~2/micros~2/windows/v10.0a/bin/NETFX4~1.8TO/gacutil.exe SN=/c/progra~2/micros~2/windows/v10.0a/bin/NETFX4~1.8TO/sn.exe
I used Short-filenames to fix the space issue for directories.
I managed to get the integration working with Delphi from the C# Integration. I called the Xapian.DLL and called stdcall extern CSharp* functions
I was trying to compile xapian-core on msys2 and I always got "xapian configure: error: no 32 bit integer type found"
This part of the instructions solved my problem. But I'm not sure what the reason for the failure was.
# In MSYS2 console
pacman -Syu
# close and reopen if prompted
pacman -Su
# Close and reopen if prompted
# if getting errors on the below you can try the --force option.
pacman -Syuu
pacman -S base-devel --needed
pacman -S mingw-w64-x86_64-toolchain zlib file pcre mingw64/mingw-w64-x86_64-file mingw64/mingw-w64-x86_64-pcre mingw-w64-x86_64-tesseract-ocr mingw-w64-x86_64-tesseract-data-eng git doxygen tcl mingw-w64-x86_64-graphviz mingw-w64-x86_64-python-docutils python3-pip python3-pygments
pacman -R libtool
pacman -S mingw64/mingw-w64-x86_64-libtool mingw-w64-x86_64-cmake gcc mingw64/mingw-w64-x86_64-zlib msys/zlib-devel msys/libuuid-devel pcre-devel
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Do you know how to compile the Windows DLL, or clues to do so?