首次在 Ubuntu 编译 bigtop 组件的环境初始化

提示

首次在 Ubuntu 22/24 上编译 bigtop 组件及相关大数据平台，必须提前准备好全部依赖环境。建议严格按照本文清单和脚本执行，避免编译时反复缺包或变量错误。

1. 依赖环境一键初始化脚本

#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: JaneTTR

set -ex
echo "############## INIT APT start #############"
FLAG_FILE="/var/lib/init/.lock"
mkdir -p /var/lib/init

# ========== repo 模板函数 ==========
write_repo_ubuntu22() {
  # $1 为 Nexus 私服地址，比如 192.168.1.100
  cat >/etc/apt/sources.list <<EOF
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-updates main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-security main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-backports main restricted universe multiverse
EOF
}

# ========== 各系统初始化操作函数 ==========
init_ubuntu22() {
  apt-get clean
  apt-get update

  echo "执行 Ubuntu 22.04 初始化脚本"
  DEBIAN_FRONTEND=noninteractive apt-get install -y \
    openssh-client \
    openssh-server \
    sudo \
    net-tools \
    unzip \
    wget \
    git \
    patch \
    build-essential \
    cmake \
    autoconf \
    automake \
    libtool \
    vim \
    lsof \
    iproute2 \
    less \
    curl \
    bzip2 \
    zlib1g-dev \
    libssl-dev \
    libsnappy-dev \
    libbz2-dev \
    libtirpc-dev \
    libkrb5-dev \
    libxml2-dev \
    protobuf-compiler \
    python3 \
    python3-pip \
    tar \
    debhelper \
    devscripts \
    build-essential \
    dh-make \
    lintian \
    fakeroot \
    locales \
    libcppunit-dev pkg-config m4 autoconf-archive \
    liblzo2-dev libzip-dev sharutils libfuse-dev \
    libsasl2-dev libgsasl-dev
  rm -rf /var/lib/apt/lists/*
}

# ========== 主流程 ==========
rm_init_repos() {
  NEXUS_IP=$(cat /scripts/system/before/nexus/.lock)
  echo "读取nexus 服务器地址：$NEXUS_IP"

  OS_ID=$(grep -oP '^ID="?(\w+)"?' /etc/os-release | cut -d= -f2 | tr -d '"')
  OS_VERSION_ID=$(grep -oP '^VERSION_ID="?([0-9\.]+)"?' /etc/os-release | cut -d= -f2 | tr -d '"')

  case "${OS_ID}_${OS_VERSION_ID}" in
    ubuntu_22.04* )
      write_repo_ubuntu22 "$NEXUS_IP"
      echo "repo 文件已写入，正在初始化 Ubuntu 22.04 repo ..."
      init_ubuntu22
      ;;
    *)
      echo "未知系统: ${OS_ID} ${OS_VERSION_ID}，请手动补充repo模板和初始化逻辑" >&2
      exit 1
      ;;
  esac
}

rm_init_repos

if [ ! -f "$FLAG_FILE" ]; then
  echo 'root:root' | chpasswd
  mkdir -p /var/run/sshd
  if ! command -v ssh-keygen >/dev/null 2>&1; then
    echo "ERROR: ssh-keygen not found! openssh-client 没装成功？"
    exit 2
  fi
  ssh-keygen -A

  # 确保 sshd_config 存在并允许 root 登录
  if [ ! -f "/etc/ssh/sshd_config" ]; then
    touch /etc/ssh/sshd_config
  fi
  grep -q '^PermitRootLogin' /etc/ssh/sshd_config \
    && sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
    || echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config

  # 常用 alias 配置
  grep -qxF "alias ll='ls -alF --color=auto'" /etc/profile || echo "alias ll='ls -alF --color=auto'" >> /etc/profile
  grep -qxF "alias ls='ls --color=auto'" /etc/profile || echo "alias ls='ls --color=auto'" >> /etc/profile
  grep -qxF "source /etc/profile" /root/.bashrc || echo "source /etc/profile" >> /root/.bashrc

  touch "$FLAG_FILE"
else
  echo "APT 已经完成初始化"
fi

echo "############## INIT APT end #############"

2. 编译链路关键依赖及用途

依赖项	主要用途与说明	参考链接
JDK	编译 Hadoop、Spark、HBase、Hive、Bigtop 等所有大数据主流组件（建议 1.8 或 17），需设置 JAVA_HOME	JDK 1.8 JDK 17
Maven	Hadoop、HBase、Hive、Ambari 等 Java 项目源码编译与依赖管理，需设置 MAVEN_HOME	Maven 3.8.4 Maven 3.9.9
Gradle	Bigtop 体系下 Spark/Flink/Trino/Superset/Redis 等自动化编译与打包，需设置 GRADLE_HOME	Gradle 5.6.4
R	Spark 编译 sparkR、R 扩展等（没有会导致 Spark 编译失败）	R 4.4.2
Python3 虚拟环境	Hadoop3 源码编译强依赖 python3（推荐3.7+） ，建议 virtualenv/venv 隔离	见下节代码

3. Python3 虚拟环境创建脚本（用于 Hadoop 编译）

#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

echo "############## SETUP PYTHON_VIRTUAL_ENV start #############"

# 设定为非交互，避免卡住任何界面
export DEBIAN_FRONTEND=noninteractive
sudo ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

BASE_DIR="/opt/modules/virtual_env"
PYTHON_VERSION=3.7.12
PYTHON_TAR=Python-$PYTHON_VERSION.tgz
PYTHON_SRC_DIR=Python-$PYTHON_VERSION
DOWNLOAD_URL=https://repo.huaweicloud.com/python/$PYTHON_VERSION/$PYTHON_TAR
PIP_URL="https://mirrors.aliyun.com/pypi/get-pip.py"
PIP_PATH="$BASE_DIR/get-pip.py"
VIRTUALENV_DIR="$BASE_DIR/hadoop_py37"

# 创建基础目录
sudo mkdir -p $BASE_DIR

# 安装所有必要依赖（防卡住）
sudo apt-get update
sudo apt-get install -y tzdata
sudo apt-get install -y build-essential zlib1g-dev libbz2-dev libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev tk-dev libgdbm-dev libpcap-dev liblzma-dev curl libffi-dev wget

# 下载 Python 3.7 源码
if [ ! -f $BASE_DIR/$PYTHON_TAR ]; then
    echo "Downloading Python $PYTHON_VERSION..."
    wget -O $BASE_DIR/$PYTHON_TAR $DOWNLOAD_URL
else
    echo "Python $PYTHON_TAR already downloaded."
fi

# 解压源码包
if [ ! -d $BASE_DIR/$PYTHON_SRC_DIR ]; then
    echo "Extracting Python $PYTHON_TAR..."
    tar -xf $BASE_DIR/$PYTHON_TAR -C $BASE_DIR
else
    echo "Python source directory already exists."
fi

cd $BASE_DIR/$PYTHON_SRC_DIR

if ! command -v python3.7 &> /dev/null; then
    echo "Installing Python $PYTHON_VERSION..."
    ./configure --enable-optimizations
    make -j$(nproc)
    sudo make altinstall
else
    echo "Python 3.7 already installed."
fi

# 安装 pip
if [ ! -f "$PIP_PATH" ]; then
    echo "Downloading get-pip.py..."
    wget -O "$PIP_PATH" "$PIP_URL"
else
    echo "get-pip file exists: $PIP_PATH"
fi

if ! command -v pip3.7 &> /dev/null; then
    echo "Installing pip..."
    sudo python3.7 "$PIP_PATH"
    pip3.7 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
    sudo python3.7 -m pip install --upgrade pip
    echo "pip 安装完毕"
else
    echo "pip already installed."
fi

pip3.7 install virtualenv

if [ ! -d $VIRTUALENV_DIR ]; then
    echo "Creating virtual environment..."
    python3.7 -m venv $VIRTUALENV_DIR
else
    echo "Virtual environment already exists."
fi

source $VIRTUALENV_DIR/bin/activate

PYTHON_VERSION_ACTUAL=$(python --version)
echo "虚拟环境中的 Python 版本是: $PYTHON_VERSION_ACTUAL"

echo "############## SETUP PYTHON_VIRTUAL_ENV end #############"