oracle-java8, hadoop2.6.4, hbase0.98.21 and mysql5.7 on ubuntu16.04-amd64

oracle-java 8

apt-get update
apt-get upgrade
#apt-get install default-jdk default-jre #openjdk-8*也可以

apt-get install software-properties-common # if ubuntu16-server-amd64
add-apt-repository ppa:webupd8team/java
apt-get update
#apt-get install oracle-java6-installer
#apt-get install oracle-java7-installer
apt-get install oracle-java8-installer

update-alternatives --config java
vi /etc/environment
JAVA_HOME=”/usr/lib/jvm/java-8-oracle”
source /etc/environment
echo $JAVA_HOME
java -version

prepare hduser:group

addgroup hadoop
adduser --ingroup hadoop hduser
adduser hduser sudo #allow sudo
sudo su hduser
sudo apt-get install openssh-server ssh
ssh-keygen -t rsa
cat ~/.ssh/id-rsa.pub >> ~/.ssh/authorized_keys

hadoop 2.6.4

disable ipv6 #Hadoop doesn’t work on IPv6
wget http://apache.stu.edu.tw/hadoop/common/hadoop-2.6.4/hadoop-2.6.4.tar.gz
tar zxvf hadoop-2.6.4.tar.gz -C /usr/local
cd /usr/local
mv hadoop-2.6.4 hadoop
chown hduser:hadoop -R /usr/local/hadoop
cd /usr/local/hadoop
vi ~/.bashrc

1
2
3
4
5
6
7
8
9
10
11
12
#HADOOP VARIABLES START
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
#HADOOP VARIABLES END

source ~/.bashrc

cd /usr/local/hadoop/etc/hadoop/

vi /usr/local/hadoop/etc/hadoop/hadoop-env.sh

1
export JAVA_HOME=/usr/lib/jvm/java-8-oracle

vi /usr/local/hadoop/etc/hadoop/core-site.xml

1
2
3
4
5
6
<configuration>
<property>
	<name>fs.default.name</name>
	<value>hdfs://localhost:9000</value>
</property>
</configuration>

cp mapred-site.xml.template mapred-site.xml
vi /usr/local/hadoop/etc/hadoop/mapred-site.xml

1
2
3
4
5
6
<configuration>
<property>
	<name>mapreduce.framework.name</name>
	<value>yarn</value>
</property>
</configuration>

vi /usr/local/hadoop/etc/hadoop/hdfs-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
<configuration>
<property>
	<name>dfs.replication</name>
	<value>1</value>
</property>
<property>
	<name>dfs.name.dir</name>
	<value>file:///home/hduser/hadoopinfra/hdfs/namenode</value>
</property>
<property>
	<name>dfs.data.dir</name>
	<value>file:///home/hduser/hadoopinfra/hdfs/datanode</value>
</property>
</configuration>

vi /usr/local/hadoop/etc/hadoop/yarn-site.xml

1
2
3
4
5
6
7
8
9
10
<configuration>
<property>
	<name>yarn.nodemanager.aux-services</name>
	<value>mapreduce_shuffle</value>
</property>
<property>
	<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
	<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
</configuration>
init

bin/hdfs namenode -format

start

sbin/start-dfs.sh
sbin/start-yarn.sh
bin/hadoop versions
jps
netstat -plten |grep java

use

bin/hadoop fs -df
bin/hadoop fs -ls /
sudo -u hduser bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar pi 2 5
#sudo -u hduser bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar pi 16 10000000

web ui

http://192.168.3.204:50070/ #hadoop services
http://192.168.3.204:8088/cluster #hadoop applications

stop

sbin/stop-yarn.sh
sbin/stop-dfs.sh

hbase 0.98.21

wget http://apache.stu.edu.tw/hbase/0.98.21/hbase-0.98.21-hadoop2-bin.tar.gz
tar zxvf hbase-0.98.21-hadoop2-bin.tar.gz -C /usr/local
cd /usr/local
mv hbase-0.98.21-hadoop2 hbase
chown hduser:hadoop -R /usr/local/hbase
cd /usr/local/hbase
vi conf/hbase-site.xml
hbase.rootdir=file:///home/hduser/hbase/hfiles or hdfs://localhost:9000/hbase
hbase.zookeeper.property.dataDir=/home/hduser/zookeeper
其它hbase.zookeeper相關改port什麼的就隨意

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
<property>
	<name>hbase.rootdir</name>
	<value>file:///home/hduser/hbase/hfiles</value>
	<!--
	 <value>hdfs://localhost:8030/hbase</value>
	-->
</property>

<property>
	<name>hbase.zookeeper.property.dataDir</name>
	<value>/home/hduser/zookeeper</value>
</property>
<property>
	<name>hbase.zookeeper.peerport</name>
	<value>2889</value>
</property>
<property>
	<name>hbase.zookeeper.leaderport</name>
	<value>2889</value>
</property>
<property>
	<name>hbase.zookeeper.property.clientPort</name>
	<value>2182</value>
</property>

vi /usr/local/hbase/conf/hbase-env.sh

1
2
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export HBASE_HOME=/usr/local/hbase
start

bin/start-hbase.sh #export HBASE_MANAGES_ZK=true #控制使用內建的zookeeper

use

jps
bin/hbase shell

1
2
3
4
5
6
7
8
9
status
list
create 'test','cf'
is_enabled 'test'
put 'test','rowkey1','cf:name','alex'
put 'test','rowkey2','cf:name','banana'
put 'test','rowke31','cf:nickname','charlie'
scan 'test'
exit
web ui

http://192.168.3.204:60010/

stop

bin/stop-hbase.sh

logs

tail logs/hbase-hduser-master-ubuntu.out
tail logs/hbase-hduser-zookeeper.ubuntu.out

mysql 5.7

apt -y update
apt -y upgrade
apt -y install mysql-server mysql-client #mysql5.7
mysql_secure_installation
systemctl status mysql
service mysql status
mysql -u root -p #enter password

1
2
3
4
5
6
7
8
status
create database test;
grant all on test.* to 'leon' identified by 'Pw12345678';
select * from mysql.user;
use test;
show tables;
show databases;
quit

vi /etc/mysql/mysql.conf.d/mysqld.cnf