TT Bigdata
JaneTTR
2025-07-16
目录
首次在 Ubuntu 编译 bigtop 组件的环境初始化
1. 依赖环境一键初始化脚本
2. 编译链路关键依赖及用途
3. Python3 虚拟环境创建脚本（用于 Hadoop 编译）

首次编译-环境初始化必装

# 首次在 Ubuntu 编译 bigtop 组件的环境初始化

提示

首次在 Ubuntu 22/24 上编译 bigtop 组件及相关大数据平台，必须提前准备好全部依赖环境。建议严格按照本文清单和脚本执行，避免编译时反复缺包或变量错误。

# 1. 依赖环境一键初始化脚本


#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: JaneTTR

set -ex
echo "############## INIT APT start #############"
FLAG_FILE="/var/lib/init/.lock"
mkdir -p /var/lib/init

# ========== repo 模板函数 ==========
write_repo_ubuntu22() {
  # $1 为 Nexus 私服地址，比如 192.168.1.100
  cat >/etc/apt/sources.list <<EOF
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-updates main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-security main restricted universe multiverse
deb [trusted=yes] http://$1:8081/repository/ubuntu22-tsinghua/ jammy-backports main restricted universe multiverse
EOF
}

# ========== 各系统初始化操作函数 ==========
init_ubuntu22() {
  apt-get clean
  apt-get update

  echo "执行 Ubuntu 22.04 初始化脚本"
  DEBIAN_FRONTEND=noninteractive apt-get install -y \
    openssh-client \
    openssh-server \
    sudo \
    net-tools \
    unzip \
    wget \
    git \
    patch \
    build-essential \
    cmake \
    autoconf \
    automake \
    libtool \
    vim \
    lsof \
    iproute2 \
    less \
    curl \
    bzip2 \
    zlib1g-dev \
    libssl-dev \
    libsnappy-dev \
    libbz2-dev \
    libtirpc-dev \
    libkrb5-dev \
    libxml2-dev \
    protobuf-compiler \
    python3 \
    python3-pip \
    tar \
    debhelper \
    devscripts \
    build-essential \
    dh-make \
    lintian \
    fakeroot \
    locales \
    libcppunit-dev pkg-config m4 autoconf-archive \
    liblzo2-dev libzip-dev sharutils libfuse-dev \
    libsasl2-dev libgsasl-dev
  rm -rf /var/lib/apt/lists/*
}

# ========== 主流程 ==========
rm_init_repos() {
  NEXUS_IP=$(cat /scripts/system/before/nexus/.lock)
  echo "读取nexus 服务器地址：$NEXUS_IP"

  OS_ID=$(grep -oP '^ID="?(\w+)"?' /etc/os-release | cut -d= -f2 | tr -d '"')
  OS_VERSION_ID=$(grep -oP '^VERSION_ID="?([0-9\.]+)"?' /etc/os-release | cut -d= -f2 | tr -d '"')

  case "${OS_ID}_${OS_VERSION_ID}" in
    ubuntu_22.04* )
      write_repo_ubuntu22 "$NEXUS_IP"
      echo "repo 文件已写入，正在初始化 Ubuntu 22.04 repo ..."
      init_ubuntu22
      ;;
    *)
      echo "未知系统: ${OS_ID} ${OS_VERSION_ID}，请手动补充repo模板和初始化逻辑" >&2
      exit 1
      ;;
  esac
}

rm_init_repos

if [ ! -f "$FLAG_FILE" ]; then
  echo 'root:root' | chpasswd
  mkdir -p /var/run/sshd
  if ! command -v ssh-keygen >/dev/null 2>&1; then
    echo "ERROR: ssh-keygen not found! openssh-client 没装成功？"
    exit 2
  fi
  ssh-keygen -A

  # 确保 sshd_config 存在并允许 root 登录
  if [ ! -f "/etc/ssh/sshd_config" ]; then
    touch /etc/ssh/sshd_config
  fi
  grep -q '^PermitRootLogin' /etc/ssh/sshd_config \
    && sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
    || echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config

  # 常用 alias 配置
  grep -qxF "alias ll='ls -alF --color=auto'" /etc/profile || echo "alias ll='ls -alF --color=auto'" >> /etc/profile
  grep -qxF "alias ls='ls --color=auto'" /etc/profile || echo "alias ls='ls --color=auto'" >> /etc/profile
  grep -qxF "source /etc/profile" /root/.bashrc || echo "source /etc/profile" >> /root/.bashrc

  touch "$FLAG_FILE"
else
  echo "APT 已经完成初始化"
fi

echo "############## INIT APT end #############"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

# 2. 编译链路关键依赖及用途

依赖项 主要用途与说明 参考链接
JDK 编译 Hadoop、Spark、HBase、Hive、Bigtop
等所有大数据主流组件（建议 1.8 或 17），需设置 JAVA_HOME		 JDK 1.8
JDK 17
Maven Hadoop、HBase、Hive、Ambari 等
Java 项目源码编译与依赖管理，需设置 MAVEN_HOME		 Maven 3.8.4
Maven 3.9.9
Gradle Bigtop 体系下 Spark/Flink/Trino/Superset/Redis
等自动化编译与打包，需设置 GRADLE_HOME		 Gradle 5.6.4
R Spark 编译 sparkR、R 扩展等（没有会导致 Spark 编译失败） R 4.4.2
Python3 虚拟环境 Hadoop3 源码编译强依赖 python3（推荐3.7+）
，建议 virtualenv/venv 隔离		 见下节代码

# 3. Python3 虚拟环境创建脚本（用于 Hadoop 编译）

#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

echo "############## SETUP PYTHON_VIRTUAL_ENV start #############"

# 设定为非交互，避免卡住任何界面
export DEBIAN_FRONTEND=noninteractive
sudo ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

BASE_DIR="/opt/modules/virtual_env"
PYTHON_VERSION=3.7.12
PYTHON_TAR=Python-$PYTHON_VERSION.tgz
PYTHON_SRC_DIR=Python-$PYTHON_VERSION
DOWNLOAD_URL=https://repo.huaweicloud.com/python/$PYTHON_VERSION/$PYTHON_TAR
PIP_URL="https://mirrors.aliyun.com/pypi/get-pip.py"
PIP_PATH="$BASE_DIR/get-pip.py"
VIRTUALENV_DIR="$BASE_DIR/hadoop_py37"

# 创建基础目录
sudo mkdir -p $BASE_DIR

# 安装所有必要依赖（防卡住）
sudo apt-get update
sudo apt-get install -y tzdata
sudo apt-get install -y build-essential zlib1g-dev libbz2-dev libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev tk-dev libgdbm-dev libpcap-dev liblzma-dev curl libffi-dev wget

# 下载 Python 3.7 源码
if [ ! -f $BASE_DIR/$PYTHON_TAR ]; then
    echo "Downloading Python $PYTHON_VERSION..."
    wget -O $BASE_DIR/$PYTHON_TAR $DOWNLOAD_URL
else
    echo "Python $PYTHON_TAR already downloaded."
fi

# 解压源码包
if [ ! -d $BASE_DIR/$PYTHON_SRC_DIR ]; then
    echo "Extracting Python $PYTHON_TAR..."
    tar -xf $BASE_DIR/$PYTHON_TAR -C $BASE_DIR
else
    echo "Python source directory already exists."
fi

cd $BASE_DIR/$PYTHON_SRC_DIR

if ! command -v python3.7 &> /dev/null; then
    echo "Installing Python $PYTHON_VERSION..."
    ./configure --enable-optimizations
    make -j$(nproc)
    sudo make altinstall
else
    echo "Python 3.7 already installed."
fi

# 安装 pip
if [ ! -f "$PIP_PATH" ]; then
    echo "Downloading get-pip.py..."
    wget -O "$PIP_PATH" "$PIP_URL"
else
    echo "get-pip file exists: $PIP_PATH"
fi

if ! command -v pip3.7 &> /dev/null; then
    echo "Installing pip..."
    sudo python3.7 "$PIP_PATH"
    pip3.7 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
    sudo python3.7 -m pip install --upgrade pip
    echo "pip 安装完毕"
else
    echo "pip already installed."
fi

pip3.7 install virtualenv

if [ ! -d $VIRTUALENV_DIR ]; then
    echo "Creating virtual environment..."
    python3.7 -m venv $VIRTUALENV_DIR
else
    echo "Virtual environment already exists."
fi

source $VIRTUALENV_DIR/bin/activate

PYTHON_VERSION_ACTUAL=$(python --version)
echo "虚拟环境中的 Python 版本是: $PYTHON_VERSION_ACTUAL"

echo "############## SETUP PYTHON_VIRTUAL_ENV end #############"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
#Ubuntu#国产化适配#Bigtop#DEB#环境准备
解决-devscripts依赖缺失
解读-bigtop-select支持deb

← 解读-bigtop-select支持deb→

最近更新
01
当前版本 2026/08
08-29
02
把 Ambari Server HA 稳稳地跑起来
08-17
03
从一次启用操作追到主备就绪
08-17
更多文章>
Theme by Vdoing | Copyright © 2017-2026 JaneTTR | MIT License