Skip to content

Commit

Permalink
[#21] changes made to make mungefs into an external
Browse files Browse the repository at this point in the history
  • Loading branch information
@jassigill2000 @trel
jassigill2000 authored and trel committed Nov 14, 2019
1 parent 76b954a commit e08ee18
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 76 deletions.
12 changes: 10 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
all : autoconf avro aws-sdk-cpp boost clang clang-runtime cmake cpython imagemagick jansson json libarchive libs3 qpid qpid-proton qpid-with-proton redis zeromq4-1 cppzmq epm
all : autoconf avro aws-sdk-cpp boost clang clang-runtime cmake cpython imagemagick jansson json libarchive libs3 mungefs qpid qpid-proton qpid-with-proton redis zeromq4-1 cppzmq epm

server : avro boost clang-runtime cppzmq jansson libarchive zeromq4-1

Expand Down Expand Up @@ -117,6 +117,14 @@ libs3_clean :
@rm -rf libs3*
@rm -rf $(LIBS3_PACKAGE)

$(MUNGEFS_PACKAGE) : $(CPPZMQ_PACKAGE) $(LIBARCHIVE_PACKAGE) $(AVRO_PACKAGE) $(CLANG-RUNTIME_PACKAGE) $(ZEROMQ4-1_PACKAGE)
./build.py $(VERBOSITY) mungefs > mungefs.log 2>&1
mungefs : $(MUNGEFS_PACKAGE)
mungefs_clean :
@echo "Cleaning mungefs..."
@rm -rf mungefs*
@rm -rf $(MUNGEFS_PACKAGE)

$(QPID_PACKAGE) : $(CLANG_PACKAGE) $(BOOST_PACKAGE) $(QPID-PROTON_PACKAGE)
./build.py $(VERBOSITY) qpid > qpid.log 2>&1
qpid : $(QPID_PACKAGE)
Expand Down Expand Up @@ -173,7 +181,7 @@ epm_clean :
@rm -rf epm*
@rm -rf $(EPM_PACKAGE)

clean : autoconf_clean avro_clean aws-sdk-cpp_clean boost_clean clang_clean clang-runtime_clean cmake_clean cpython_clean jansson_clean libarchive_clean libs3_clean zeromq4-1_clean cppzmq_clean epm_clean
clean : autoconf_clean avro_clean aws-sdk-cpp_clean boost_clean clang_clean clang-runtime_clean cmake_clean cpython_clean jansson_clean libarchive_clean libs3_clean mungefs_clean zeromq4-1_clean cppzmq_clean epm_clean
@echo "Cleaning generated files..."
@rm -rf packages.mk
@echo "Done."
74 changes: 71 additions & 3 deletions build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,38 @@ def touch(filename):
except:
open(filename, 'a').close()

def get_rvm_path():
cmd = ['whereis', 'rvm']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
_out, _err = p.communicate()
index = len(_out.lstrip().split(': ')) - 1
rvm_path = _out.lstrip().split(': ')[index]
return rvm_path.strip()

def set_environ_path(bin_path):
path = os.environ['PATH']
new_path = bin_path + ':' + path
os.environ['PATH'] = new_path

def set_rvm_path():
rvm_path = get_rvm_path()
rvm_bin = os.path.join(rvm_path, 'bin')
set_environ_path(rvm_bin)

def set_ruby_path():
rvm_path = '/usr/local/rvm'
ruby_path= os.path.join(rvm_path, 'rubies/ruby-2.6.3')
ruby_bin = os.path.join(ruby_path, 'bin')
os.environ['GEM_HOME'] = ruby_path
set_environ_path(ruby_bin)

def set_clang_path():
p = get_versions()['clang']
path_name = '{0}{1}-{2}'.format('clang', p['version_string'], p['consortium_build_number'])
root_dir = os.getcwd().split('/')[1]
clang_binpath = '/{0}/{1}/bin'.format(root_dir, path_name)
set_environ_path(clang_binpath)

def get_local_path(package_name, path_elements):
log = logging.getLogger(__name__)
p = get_versions()[package_name]
Expand All @@ -40,8 +72,10 @@ def get_local_path(package_name, path_elements):
def run_cmd(cmd, run_env=False, unsafe_shell=False, check_rc=False):
log = logging.getLogger(__name__)
# run it
if run_env == False:
run_env = os.environ.copy()
if run_env == True:
set_rvm_path()

run_env = os.environ.copy()
log.debug('run_env: {0}'.format(run_env))
log.info('running: {0}, unsafe_shell={1}, check_rc={2}'.format(cmd, unsafe_shell, check_rc))
if unsafe_shell == True:
Expand Down Expand Up @@ -144,6 +178,12 @@ def build_package(target):
log.debug('cmake_executable: [{0}]'.format(cmake_executable))

# prepare libraries
cppzmq_root = get_local_path('cppzmq',[])
log.debug('cppzmq_root: [{0}]'.format(cppzmq_root))
zmq_root = get_local_path('zeromq4-1',[])
log.debug('zmq_root: [{0}]'.format(zmq_root))
avro_root = get_local_path('avro',[])
log.debug('avro_root: [{0}]'.format(avro_root))
boost_root = get_local_path('boost',[])
log.debug('boost_root: [{0}]'.format(boost_root))

Expand All @@ -160,9 +200,25 @@ def build_package(target):
boost_install_prefix = os.path.join(boost_info['externals_root'], boost_subdirectory)
boost_rpath = os.path.join(boost_install_prefix, 'lib')

avro_info = get_versions()['avro']
avro_subdirectory = '{0}{1}-{2}'.format('avro', avro_info['version_string'], avro_info['consortium_build_number'])
avro_install_prefix = os.path.join(avro_info['externals_root'], avro_subdirectory)
avro_rpath = os.path.join(avro_install_prefix, 'lib')

libarchive_info = get_versions()['libarchive']
libarchive_subdirectory = '{0}{1}-{2}'.format('libarchive', libarchive_info['version_string'], libarchive_info['consortium_build_number'])
libarchive_install_prefix = os.path.join(libarchive_info['externals_root'], libarchive_subdirectory)
libarchive_rpath = os.path.join(libarchive_install_prefix, 'lib')

zmq_info = get_versions()['zeromq4-1']
zmq_subdirectory = '{0}{1}-{2}'.format('zeromq4-1', zmq_info['version_string'], zmq_info['consortium_build_number'])
zmq_install_prefix = os.path.join(zmq_info['externals_root'], zmq_subdirectory)
zmq_rpath = os.path.join(zmq_install_prefix, 'lib')

clang_info = get_versions()['clang']
clang_subdirectory = '{0}{1}-{2}'.format('clang', clang_info['version_string'], clang_info['consortium_build_number'])
clang_executable = os.path.join(script_path, '{0}'.format(clang_subdirectory), 'bin', 'clang++')
clang_executable = os.path.join(script_path, '{0}'.format(clang_subdirectory), 'bin', 'clang')
clangpp_executable = os.path.join(script_path, '{0}'.format(clang_subdirectory), 'bin', 'clang++')
clang_cpp_headers = os.path.join(script_path, '{0}'.format(clang_subdirectory), 'include', 'c++', 'v1')
clang_cpp_libraries = os.path.join(script_path, '{0}'.format(clang_subdirectory), 'lib')

Expand Down Expand Up @@ -218,6 +274,9 @@ def build_package(target):
run_cmd(['git', 'checkout', v['commitish']], check_rc='git checkout failed')

# set environment
if target == 'boost':
set_clang_path()

myenv = os.environ.copy()
if target not in ['clang','cmake','autoconf','cpython']:
clang_bindir = get_local_path('clang',['bin'])
Expand Down Expand Up @@ -248,6 +307,7 @@ def build_package(target):
i = re.sub("TEMPLATE_CLANG_CPP_LIBRARIES", clang_cpp_libraries, i)
i = re.sub("TEMPLATE_CLANG_SUBDIRECTORY", clang_subdirectory, i)
i = re.sub("TEMPLATE_CLANG_EXECUTABLE", clang_executable, i)
i = re.sub("TEMPLATE_CLANGPP_EXECUTABLE", clangpp_executable, i)
i = re.sub("TEMPLATE_CLANG_RUNTIME_RPATH", clang_runtime_rpath, i)
i = re.sub("TEMPLATE_CMAKE_EXECUTABLE", cmake_executable, i)
i = re.sub("TEMPLATE_QPID_SUBDIRECTORY", qpid_subdirectory, i)
Expand All @@ -256,6 +316,12 @@ def build_package(target):
i = re.sub("TEMPLATE_BOOST_ROOT", boost_root, i)
i = re.sub("TEMPLATE_LIBS3_MAKEFILE_STRING", libs3_makefile_string, i)
i = re.sub("TEMPLATE_BOOST_RPATH", boost_rpath, i)
i = re.sub("TEMPLATE_LIBARCHIVE_RPATH", libarchive_rpath, i)
i = re.sub("TEMPLATE_AVRO_RPATH", avro_rpath, i)
i = re.sub("TEMPLATE_AVRO_PATH", avro_root, i)
i = re.sub("TEMPLATE_ZMQ_RPATH", zmq_rpath, i)
i = re.sub("TEMPLATE_ZMQ_PATH", zmq_root, i)
i = re.sub("TEMPLATE_CPPZMQ_PATH", cppzmq_root, i)
run_cmd(i, run_env=myenv, unsafe_shell=True, check_rc='build failed')

# MacOSX - after building boost
Expand Down Expand Up @@ -356,6 +422,8 @@ def main():
for p in v:
f.write('{0}_PACKAGE={1}\n'.format(p.upper(), get_package_filename(p)))
elif target in get_versions():
set_rvm_path()
set_ruby_path()
build_package(target)
else:
log.error('build target [{0}] not found in {1}'.format(target, sorted(get_versions().keys())))
Expand Down
92 changes: 26 additions & 66 deletions install_prerequisites.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,20 @@ def mkdir_p(path):
else:
raise

def install_rvm_and_ruby():
cmd = 'gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB'
build.run_cmd(cmd, unsafe_shell=True,check_rc='gpg keys not received')
cmd = 'curl -sSL https://get.rvm.io | bash -s stable'
build.run_cmd(cmd, unsafe_shell=True, check_rc='curl failed')
cmd = 'rvm reload && rvm requirements run && rvm install 2.6'
build.run_cmd(cmd, unsafe_shell=True, run_env=True, check_rc='rvm ruby install failed')

def install_fpm_gem():
build.set_ruby_path()
cmd = 'rvm reload && rvm use 2.6 && gem install -v 1.4.0 fpm'
build.run_cmd(cmd, unsafe_shell=True, run_env=True, check_rc='fpm gem install failed')


def main():
# configure parser
parser = optparse.OptionParser()
Expand Down Expand Up @@ -46,98 +60,39 @@ def main():
# get prerequisites
cmd = ['sudo','apt-get','install','-y','automake','make','autoconf2.13','texinfo',
'help2man','g++','git','lsb-release','libtool','python-dev','libbz2-dev','zlib1g-dev',
'libcurl4-gnutls-dev','libxml2-dev','pkg-config','uuid-dev','libssl-dev']
if pld in ['Ubuntu'] and platform.linux_distribution()[1] < '14':
cmd.extend(['ruby1.9.1','ruby1.9.1-dev',])
else:
cmd.extend(['ruby','ruby-dev',])
'libcurl4-gnutls-dev','libxml2-dev','pkg-config','uuid-dev','libssl-dev','libfuse-dev']
build.run_cmd(cmd, check_rc='installing prerequisites failed')
# if old, bootstrap g++
if pld in ['Ubuntu'] and platform.linux_distribution()[1] < '14':
# ubuntu12 ships with g++ 4.6 - needs 4.8+ to build clang
log.info('Detected: Old Ubuntu - need to get g++ 4.8 to build clang')
cmd = ['sudo','apt-get','install','-y','python-software-properties']
build.run_cmd(cmd, check_rc='installing add-apt-repository prereq failed')
cmd = ['sudo', 'add-apt-repository', '-y', 'ppa:ubuntu-toolchain-r/test']
build.run_cmd(cmd, check_rc='installing ppa failed')
cmd = ['sudo', 'apt-get', 'update', '-y']
build.run_cmd(cmd, check_rc='getting updates failed')
cmd = ['sudo', 'apt-get', 'install', '-y', 'g++-4.8']
build.run_cmd(cmd, check_rc='installing g++-4.8 failed')
cmd = ['sudo', 'update-alternatives', '--install', '/usr/bin/g++', 'g++', '/usr/bin/g++-4.8', '50']
build.run_cmd(cmd, check_rc='swapping g++-4.8 failed')
cmd = ['sudo', 'update-alternatives', '--install', '/usr/bin/gcc', 'gcc', '/usr/bin/gcc-4.8', '50']
build.run_cmd(cmd, check_rc='swapping gcc-4.8 failed')
# if new, get autoconf
if pld in ['Ubuntu'] and platform.linux_distribution()[1] > '16':
log.info('Detected: Ubuntu 16+ - need to get autoconf')
cmd = ['sudo','apt-get','install','-y','autoconf']
build.run_cmd(cmd, check_rc='installing autoconf failed')
# get necessary ruby gems
cmd = ['sudo','gem','install','-v','1.8.1','ffi']
build.run_cmd(cmd, check_rc='installing ffi failed')
cmd = ['sudo','gem','install','-v','1.8.3','json']
build.run_cmd(cmd, check_rc='installing json failed')
# debian needs a symlink
symlink_target = '/usr/share/rubygems-integration/all/gems/rake-10.5.0/bin/rake'
if pld in ['debian']:
if os.path.lexists(symlink_target):
os.remove(symlink_target)
mkdir_p(os.path.dirname(symlink_target))
cmd = ['sudo','ln','-s','/usr/bin/rake',symlink_target]
build.run_cmd(cmd, check_rc='preparing rake symlink failed')
cmd = ['sudo','gem','install','-v','1.4.0','fpm']
build.run_cmd(cmd, check_rc='installing fpm failed')

elif pld in ['CentOS', 'CentOS Linux', 'Red Hat Enterprise Linux Server', 'Scientific Linux']:
log.info('Detected: {0}'.format(pld))
# prep
cmd = ['sudo', 'rpm', '--rebuilddb']
build.run_cmd(cmd, check_rc='rpm rebuild failed')
cmd = ['sudo','yum','clean','all']
build.run_cmd(cmd, check_rc='yum clean failed')
cmd = ['sudo','yum','install','centos-release-scl-rh', '-y']
build.run_cmd(cmd, check_rc='yum install failed')
cmd = ['sudo','yum','update','-y','glibc*','yum*','rpm*','python*']
build.run_cmd(cmd, check_rc='yum update failed')
# get prerequisites
cmd = ['sudo','yum','install','-y','epel-release','wget','openssl','ca-certificates']
build.run_cmd(cmd, check_rc='installing epel failed')
cmd = ['sudo','yum','install','-y','gcc-c++','git','autoconf','automake','texinfo',
'help2man','rpm-build','rubygems','ruby-devel','python-devel','zlib-devel',
'help2man','rpm-build','fuse','fuse-devel','python-devel','zlib-devel',
'bzip2-devel','libcurl-devel','libxml2-devel','libtool','libuuid-devel','openssl-devel']
build.run_cmd(cmd, check_rc='installing prerequisites failed')
# get necessary ruby gems
cmd = ['sudo','gem','install','-v','1.8.1','ffi']
build.run_cmd(cmd, check_rc='installing ffi failed')
cmd = ['sudo','gem','install','-v','1.8.3','json']
build.run_cmd(cmd, check_rc='installing json failed')
cmd = ['sudo','gem','install','-v','1.4.0','fpm']
build.run_cmd(cmd, check_rc='installing fpm failed')
# if old, bootstrap g++
if platform.linux_distribution()[1] < '7':
# centos6 ships with g++ 4.4 - needs 4.8+ to build clang
log.info('Detected: Old {0} - need to get g++ 4.8 to build clang'.format(pld))
cmd = ['wget','https://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/obsolete/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern']
build.run_cmd(cmd, check_rc='wget cern key failed')
cmd = ['sudo','rpm','--import','RPM-GPG-KEY-cern']
build.run_cmd(cmd, check_rc='importing cern key failed')
cmd = ['sudo','wget','-O','/etc/yum.repos.d/slc6-devtoolset.repo','http://linuxsoft.cern.ch/cern/devtoolset/slc6-devtoolset.repo']
build.run_cmd(cmd, check_rc='wget devtoolset failed')
cmd = ['sudo','yum','install','-y','devtoolset-2']
build.run_cmd(cmd, check_rc='yum install devtoolset failed')
print('========= set environment to use the new g++ ========= ')
print('export CC=/opt/rh/devtoolset-2/root/usr/bin/gcc')
print('export CXX=/opt/rh/devtoolset-2/root/usr/bin/g++')

elif pld in ['openSUSE ', 'SUSE Linux Enterprise Server']:
log.info('Detected: {0}'.format(pld))
# get prerequisites
cmd = ['sudo','zypper','install','-y','ruby-devel','makeinfo','rubygems','libopenssl-devel',
'help2man','python-devel','libbz2-devel','libcurl-devel','libxml2-devel','uuid-devel']
build.run_cmd(cmd, check_rc='installing prerequisites failed')
# get necessary ruby gems
cmd = ['sudo','gem','install','-v','1.8.1','ffi']
build.run_cmd(cmd, check_rc='installing ffi failed')
cmd = ['sudo','gem','install','-v','1.8.3','json']
build.run_cmd(cmd, check_rc='installing json failed')
cmd = ['sudo','gem','install','-v','1.4.0','fpm']
build.run_cmd(cmd, check_rc='installing fpm failed')
else:
if platform.mac_ver()[0] != '':
log.info('Detected: {0}'.format(platform.mac_ver()[0]))
Expand All @@ -150,5 +105,10 @@ def main():
log.error('Cannot determine prerequisites for platform [{0}]'.format(pld))
return 1

# get necessary ruby gems
install_rvm_and_ruby()
install_fpm_gem()


if __name__ == '__main__':
sys.exit(main())
Loading

0 comments on commit e08ee18

Please sign in to comment.