QLever/script

This is a script for getting started with QLever along the lines of the Quickstart description

see

• Issue 562 - qlever bash script

usage

usage: ./qlever [-h|--help]
  -h|--help: show this usage
  -aw|--all_wikidata: run all steps for wikidata
  -b|--build: build qlever docker image
  -c|--clone: clone qlever
  -e|--env: show environment
  -wd|--wikidata_download: download wikidata data dump

This helper script simplifies the access to the steps outlined in https://github.com/ad-freiburg/qlever/blob/master/docs/quickstart.md

qlever

#!/bin/bash
# 
# a script for getting started with QLever
#
# see https://github.com/ad-freiburg/qlever/blob/master/docs/quickstart.md
# see https://wiki.bitplan.com/index.php/QLever
#
# 
# WF 2022-01-28
#

# we assume the script is started from the QLEVER_HOME directory
export QLEVER_HOME=$(pwd)
startTime=0
finishTime=0

#ansi colors
#http://www.csc.uvic.ca/~sae/seng265/fall04/tips/s265s047-tips/bash-using-colors.html
blue='\033[0;34m'  
red='\033[0;31m'  
green='\033[0;32m' # '\e[1;32m' is too bright for white bg.
endColor='\033[0m'

#
# a colored message 
#   params:
#     1: l_color - the color of the message
#     2: l_msg - the message to display
#
color_msg() {
  local l_color="$1"
  local l_msg="$2"
  echo -e "${l_color}$l_msg${endColor}"
}

#
# error
#
#   show an error message and exit
#
#   params:
#     1: l_msg - the message to display
error() {
  local l_msg="$1"
  # use ansi red for error
  color_msg $red "Error: $l_msg" 1>&2
  exit 1
}

#
# show the usage
#
usage() {
  echo "usage: $0 [-h|--help]"
  echo "  -h|--help: show this usage"
  echo "  -aw|--all_wikidata: run all steps for wikidata"
  echo "  -b|--build: build qlever docker image"
  echo "  -c|--clone: clone qlever"
  echo "  -e|--env: show environment"
  echo "  -wd|--wikidata_download: download wikidata data dump"
  echo ""
  echo "This helper script simplifies the access to the steps outlined in https://github.com/ad-freiburg/qlever/blob/master/docs/quickstart.md"
}

#
# show the start of an action
#
show_timing() {
  local l_action="$1"
  local l_state="$2"
  now=$(date)
  case $l_state in
    started) 
       startTime=$SECONDS
       after=""
       ;;
    finished) 
      finishTime=$SECONDS
      local l_duration=$(( $finishTime - $startTime ))
      after=" after $l_duration seconds"
      ;;
  esac
  color_msg $blue "$l_action $l_state at $now$after"
}

#
# show the environment
#
show_env() {
  color_msg $blue "operating system"
  lsb_release -a
  color_msg $blue "docker version"
  docker --version
  color_msg $blue "memory"
  free -h
  color_msg $blue "diskspace"
  df -h | grep /dev/s
}

#
# clone the qlever code
#
qlever_clone() {
  cd $QLEVER_HOME
  if [ ! -d qlever-code ]
  then
    color_msg $blue "cloning qlever - please wait typically 1 min ..."
    show_timing "cloning qlever" "started"
    git clone --recursive https://github.com/ad-freiburg/qlever qlever-code
    show_timing "cloning qlever" "finished"
  else
    color_msg $green "clone of clever-code already available"
  fi
}

#
# build the docker image
#
qlever_build() {
  #docker images | grep qlever
  #if [ $? -ne 0 ]
  #then
     cd $QLEVER_HOME/qlever-code
     color_msg $blue "building qlever - please wait typically 15 min ..."
     show_timing "docker build" "started"
     # docker build -t qlever .
     docker build --file Dockerfiles/Dockerfile.Ubuntu20.04 -t qlever .
     show_timing "docker build" "finished"
  #else
  #  color_msg $green "qlever image already build"
  #fi
}

#
# generic download
# 
# params
#   1: title of the download
#   2: expected time
#   3: url to download from
#
download() {
  local l_title="$1"
  local l_expected="$2"
  local l_url="$3"
  color_msg $blue "downloading $l_title ... please wait typically $l_expected ..."
  show_timing "$l_title download" "started"
  wget $l_url 
  show_timing "$l_title download" "finished"
}

#
# wikidata download
#
wikidata_download() {
  cd $QLEVER_HOME
  target=qlever-indices/wikidata
  if [ ! -d $target ]
  then
     mkdir -p $target
     cd $target
     config=$QLEVER_HOME/qlever-code/examples/wikidata.settings.json
     color_msg $blue "copying config file $config"
     cp -p $config .
     download "wikidata lexemes" "3min" https://dumps.wikimedia.org/wikidatawiki/entities/latest-lexemes.ttl.bz2
     download "wikidata dump" "6hours" https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.ttl.bz2;date
  else
     color_msg $green "wikidata dump already downloaded"
  fi
}

#
# build the wikidata index
#
wikidata_index() {
   cd $QLEVER_HOME
   chmod o+w . 
   show_timing "creating wikidata index" "started"
   docker run -it --rm -v $QLEVER_HOME/qlever-indices/wikidata:/index --entrypoint bash qlever -c "cd /index && bzcat latest-all.ttl.bz2 latest-lexemes.ttl.bz2 | IndexBuilderMain -F ttl -f - -l -i wikidata -s wikidata.settings.json | tee wikidata.index-log.txt"
   show_timing "creating wikidata index" "finished"
}

# commandline options according to usage
while [  "$1" != ""  ]
do
  option=$1
  shift
  case $option in
    -h|--help)
       usage;;
    -aw|--all_wikidata)
      show_env
      qlever_clone
      qlever_build
      wikidata_download
      wikidata_index
      ;;
    -b|--build)
       qlever_build;;
    -c|--clone)
       qlever_clone;;
    -e|--env)
       show_env;;
    -wd|--wikidata_download)
       wikidata_download;;
    -wi|--wikidata-index)
       wikidata_index;;
    -t) 
      show_timing "testing" "started"
      sleep 2
      show_timing "testing" "finished"
     ;;
  esac
done