Wikidata Import 2025-11-23

From BITPlan Wiki
Revision as of 14:30, 23 November 2025 by Wf (talk | contribs) (→‎Setup)
Jump to navigation Jump to search

Import

Import
edit
state  ?
url  https://wiki.bitplan.com/index.php/Wikidata_Import_2025-11-23
target  Millenium DB
start  2025-11-23
end  
days  7
os  Ubuntu 22.04.3 LTS
cpu  Intel(R) Xeon(R) Gold 6326 CPU @ 2.90GHz (16 cores)
ram  512
triples  
comment  Try replicating https://www.wikidata.org/wiki/Wikidata:Scaling_Wikidata/Benchmarking/MilleniumDB

Millenium DB

Links

Setup

setup_milleniumdb.sh

#!/bin/bash
# MilleniumDB installation script
# Created by Gemini 3 Pro Preview
# based on https://github.com/MillenniumDB/MillenniumDB/wiki/Setup
# initiated by  WF 2025-11-23
# for https://wiki.bitplan.com/index.php/Wikidata_Import_2025-11-23
# Links
#
# https://github.com/MillenniumDB/MillenniumDB
# https://github.com/MillenniumDB/MillenniumDB/wiki
# https://www.wikidata.org/wiki/Wikidata:Scaling_Wikidata/Benchmarking/MilleniumDB
# https://www.wikidata.org/wiki/Q118954995
# 
set -e
BASE_DIR="${HOME}"
N_CORES=${1:-$(nproc)}

function install_dependencies() {
    echo "Installing dependencies..."
    sudo apt update
    sudo apt install -y git g++ cmake libssl-dev libncurses-dev less python3 python3-venv libicu-dev
}

function clone_repo() {
    echo "Cloning MillenniumDB repository..."
    mkdir -p "${BASE_DIR}/source/cpp"
    cd "${BASE_DIR}/source/cpp"
    if [ -d "MillenniumDB" ]; then
        echo "Repository already cloned. Skipping clone."
    else
        git clone https://github.com/MillenniumDB/MillenniumDB.git
    fi
    cd MillenniumDB
    export MDB_HOME=$(pwd)
    echo "MDB_HOME set to ${MDB_HOME}"
}

function install_boost() {
    echo "Installing Boost 1.82.0 locally..."
    cd "${MDB_HOME}"
    BOOST_INSTALL_PATH="${MDB_HOME}/third_party/boost_1_82/include"
    if [ -d "${BOOST_INSTALL_PATH}/boost" ]; then
        echo "Boost headers already installed. Skipping."
    else
        BOOST_VERSION_URL="1.82.0"
        BOOST_VERSION_UNDERSCORE="1_82_0"
        BOOST_DIR_NAME="boost_${BOOST_VERSION_UNDERSCORE}"
        BOOST_TARBALL="${BOOST_DIR_NAME}.tar.gz"
        BOOST_URL="https://archives.boost.io/release/${BOOST_VERSION_URL}/source/${BOOST_TARBALL}"

        wget -q --show-progress -O "${BOOST_TARBALL}" "${BOOST_URL}"
        tar -xf "${BOOST_TARBALL}"
        mkdir -p "${BOOST_INSTALL_PATH}"
        mv "${BOOST_DIR_NAME}/boost" "${BOOST_INSTALL_PATH}/"
        rm -r "${BOOST_TARBALL}" "${BOOST_DIR_NAME}"
    fi
}

function build_project() {
    echo "Building MillenniumDB using ${N_CORES} cores..."
    cd "${MDB_HOME}"
    cmake -B build/Release -D CMAKE_BUILD_TYPE=Release
    cmake --build build/Release -j "${N_CORES}"
    echo "Build complete."
}

function verify_build() {
    echo "Verifying build by running help command..."
    "${MDB_HOME}/build/Release/bin/mdb" help
}

# Main script execution flow
install_dependencies
clone_repo
install_boost
build_project
verify_build

echo "MillenniumDB setup and build completed successfully!"
echo "To use 'mdb', export MDB_HOME:"
echo "  export MDB_HOME=${MDB_HOME}"
echo "Add this line to your shell startup file for permanence."
Retrieved from "https://wiki.bitplan.com/index.php?title=Wikidata_Import_2025-11-23&oldid=20305"