Adding a new node in Oracle RAC 11gR2 using Virtual Box

Adding a new node in Oracle RAC 11gR2 using Virtual Box:

Create new Virtual Machine using Virtual Box and install same Linux version same as existing nodes RAC1 and RAC2

After installing Linux Follow the steps for Node RAC3

#Add or amend the following lines to the “/etc/sysctl.conf” file.

fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 2097152
kernel.shmmax = 1054504960
kernel.shmmni = 4096
# semaphores: semmsl, semmns, semopm, semmni
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default=262144
net.core.rmem_max=4194304
net.core.wmem_default=262144
net.core.wmem_max=1048586

#Run the following command to change the current kernel parameters.
/sbin/sysctl -p

#Add the following lines to the “/etc/security/limits.conf” file.
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 4096
oracle hard nofile 65536
oracle soft stack 10240

#Add the following lines to the “/etc/pam.d/login” file, if it does not already exist.
session required pam_limits.so

#Create the new groups and users.
Delete existing users and groups in RAC3
groupadd -g 1000 oinstall
groupadd -g 1200 dba
useradd -u 1100 -g oinstall -G dba oracle
passwd oracle

#Add .bash_profile to the oracle user
# Oracle Settings
TMP=/tmp; export TMP
TMPDIR=$TMP; export TMPDIR
ORACLE_HOSTNAME=rac3.oracle.com; export ORACLE_HOSTNAME
ORACLE_UNQNAME=RAC; export ORACLE_UNQNAME
ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE
GRID_HOME=/u01/app/11.2.0/grid; export GRID_HOME
DB_HOME=$ORACLE_BASE/product/11.2.0/db_1; export DB_HOME
ORACLE_HOME=$DB_HOME; export ORACLE_HOME
ORACLE_SID=RAC3; export ORACLE_SID
ORACLE_TERM=xterm; export ORACLE_TERM
BASE_PATH=/usr/sbin:$PATH; export BASE_PATH
PATH=$ORACLE_HOME/bin:$BASE_PATH; export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib; export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib; export CLASSPATH

if [ $USER = “oracle” ]; then
if [ $SHELL = “/bin/ksh” ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
fi

#Install the following package from the Oracle grid media after you’ve defined groups.
cd /u01/grid/rpm
rpm -Uvh cvuqdisk*

#Install compatible RPMS from CD on new node
rpm -qa compat*
rpm -ivh compat-* –force –nodeps

#Put the information of nodes in /etc/hosts on all nodes
vi /etc/hosts
# Public
192.168.1.111 rac1.oracle.com rac1
192.168.1.112 rac2.oracle.com rac2
192.168.1.120 rac3.oracle.com rac3
# Private
192.168.0.111 rac1-priv.oracle.com rac1-priv
192.168.0.112 rac2-priv.oracle.com rac2-priv
192.168.0.113 rac3-priv.oracle.com rac3-priv
# Virtual
192.168.1.113 rac1-vip.oracle.com rac1-vip
192.168.1.114 rac2-vip.oracle.com rac2-vip
192.168.1.115 rac3-vip.oracle.com rac3-vip
# SCAN
192.168.1.115 rac-scan.oracle.com rac-scan
192.168.1.116 rac-scan.oracle.com rac-scan
192.168.1.117 rac-scan.oracle.com rac-scan

:wq! -> save and exit

#Create directory structure
mkdir -p /u01/app/11.2.0/grid
mkdir -p /u01/app/oracle/product/11.2.0/db_1
chown -R oracle:oinstall /u01
chmod -R 775 /u01/

#shutdown all 3 nodes and come to windows cmd
PATH=%PATH%;C:\Program Files\Oracle\VirtualBox
cd D:\rac1

VBoxManage storageattach RAC3 –storagectl “SATA Controller” –port 1 –device 0 –type hdd –medium asm1.vdi –mtype shareable
VBoxManage storageattach RAC3 –storagectl “SATA Controller” –port 2 –device 0 –type hdd –medium asm2.vdi –mtype shareable
VBoxManage storageattach RAC3 –storagectl “SATA Controller” –port 3 –device 0 –type hdd –medium asm3.vdi –mtype shareable
VBoxManage storageattach RAC3 –storagectl “SATA Controller” –port 4 –device 0 –type hdd –medium asm4.vdi –mtype shareable

#Configure secure shell for oracle user on all nodes

ON Rac3
#su – oracle
$ssh-keygen -t rsa
(give 3 times enter)
$cd .ssh
$cat id_rsa.pub >> authorized_keys
On Rac1
$su – oracle
$cd .ssh
$scp authorized_keys oracle@rac3:/home/oracle
passwd:****
On Rac3
$pwd (/home/oracle)
$cat authorized_keys >> .ssh/authorized_keys
$cd .ssh
$scp authorized_keys oracle@rac1:/home/oracle/.ssh
$scp authorized_keys oracle@rac2:/home/oracle/.ssh

check for ssh connectivities between these 3 nodes
Run the following from rac1, rac2, rac3.
$ssh rac1 date
$ssh rac1-priv date
$ssh rac2 date
$ssh rac2-priv date
$ssh rac3 date
$ssh rac3-priv date

9) Create ASM Diskgroups
vi /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL==”sdb1″, BUS==”scsi”, PROGRAM==”/sbin/scsi_id -g -u -s /block/$parent”, RESULT==”SATA_VBOX_HARDDISK_VB6c59863c-6b11e27b_”, NAME=”asm-disk1″, OWNER=”oracle”, GROUP=”dba”, MODE=”0660″
KERNEL==”sdc1″, BUS==”scsi”, PROGRAM==”/sbin/scsi_id -g -u -s /block/$parent”, RESULT==”SATA_VBOX_HARDDISK_VBd9fc08f1-88dbd146_”, NAME=”asm-disk2″, OWNER=”oracle”, GROUP=”dba”, MODE=”0660″
KERNEL==”sdd1″, BUS==”scsi”, PROGRAM==”/sbin/scsi_id -g -u -s /block/$parent”, RESULT==”SATA_VBOX_HARDDISK_VB10dffe36-9cd70e2c_”, NAME=”asm-disk3″, OWNER=”oracle”, GROUP=”dba”, MODE=”0660″
KERNEL==”sde1″, BUS==”scsi”, PROGRAM==”/sbin/scsi_id -g -u -s /block/$parent”, RESULT==”SATA_VBOX_HARDDISK_VBb29398e9-2184d827_”, NAME=”asm-disk4″, OWNER=”oracle”, GROUP=”dba”, MODE=”0660″

Load updated block device partition tables.

# /sbin/partprobe /dev/sdb1
# /sbin/partprobe /dev/sdc1
# /sbin/partprobe /dev/sdd1
# /sbin/partprobe /dev/sde1

Test the rules are working as expected.

# udevtest /block/sdb/sdb1
# udevtest /block/sdc/sdc1
# udevtest /block/sdd/sdd1
# udevtest /block/sde/sde1

Restart the UDEV service.

# /sbin/udevcontrol reload_rules
# /sbin/start_udev

# cd /dev
# ls -al asm-disk*
brw-rw—- 1 oracle dba 8, 17 Apr 8 22:47 asm-disk1
brw-rw—- 1 oracle dba 8, 33 Apr 8 22:47 asm-disk2
brw-rw—- 1 oracle dba 8, 49 Apr 8 22:47 asm-disk3
brw-rw—- 1 oracle dba 8, 65 Apr 8 22:47 asm-disk4

#Setup NTPD before cluvfy
service ntpd stop
chkconfig ntpd off
mv /etc/ntp.conf /etc/ntp.conf.orig
rm /var/run/ntpd.pid
vi /etc/sysconfig/ntpd
Add -x to the existing line “OPTIONS=”-x -u ntp:ntp -p /var/run/ntpd.pid””
service ntpd restart

#Run cluvfy utility from any one of nodes,from oracle user
$cd /u01/app/11.2.0/grid/bin
$./cluvfy stage -pre crsinst -n rac1,rac2,rac3 -verbose

#Steps from RAC1 to add RAC3
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$ORACLE_HOME/bin:$PATH
cd /u01/app/11.2.0/grid/oui/bin
./addNode.sh -silent “CLUSTER_NEW_NODES={RAC3}” “CLUSTER_NEW_VIRTUAL_HOSTNAMES={rac3-vip.oracle.com}”

Once it is over, it will ask us to run 2 scripts in RAC3 as a root user, run it.

#Go to RDBMS Oracle Home
cd $ORACLE_HOME – it should point to RDBMS Home
cd oui/bin
./addNode.sh -silent “CLUSTER_NEW_NODES={rac3}”

#Run DBCA to add instance from RAC1
cd $ORACLE_HOME/bin
./dbca
“Oracle Real Application Clusters Database” and then “select Instance Management” and then select “add instance”

Finish

Nagulu Polagani

"We are all apprentices in a craft where no one ever becomes a master."