Here is a detailed setup for apache2/jboss4.0.5 failover cluster of two:
Hosts used for setup:
host1 aka H1
host2 aka H2
1. Configured Apache via httpd.conf (identical for both H1 and v2):
a. Within the httpd.conf add the following inside the IfModule status block:
# Add jkstatus for managing runtime data
<Location /jkstatus/>
JkMount status
Order deny,allow
Deny from all
Allow from 127.0.0.1
</Location>
b. Ensure that the following is in your jk_module If Block:
JkWorkersFile /usr/local/apache/conf/workers.properties
JkLogFile /usr/local/apache/logs/mod_jk.log
JkMountFile conf/uriworkermap.properties
JkLogLevel info # change to debug is setting up
# JkOptions indicates to send SSK KEY SIZE
JkOptions +ForwardKeySize +ForwardURICompat -ForwardDirectories
# JkRequestLogFormat
JkRequestLogFormat "%w %V %T"
# Add shared memory.
# This directive is present with 1.2.10 and
# later versions of mod_jk and is needed for
# load balancing to work properly
JkShmFile logs/jk.shm
# Make sure application Mount to your loadbalancer instead of localhost. For now assuming "balancer"
# Note: No need to do application/* because done in uriworkermap.properties but if you have Jboss serving
# your
jsp's etc you need to ensure the following as well
JkMount /*.jsp balanced
........................
....................
2. Configure workers.properties. Assuming mod_jk 1.2.27 (identical for both H1 and H2):
a. Add the new "loadbalancer" to existing workers list. Assuming "balanced"
worker.list=balanced
b. Create a root template that all the other workers (node1 and node2) can reference:
worker.template.port=8009
worker.template.type=ajp13
worker.template.lbfactor=1
worker.template.socket_connect_timeout=10
worker.template.ping_timeout=10000
worker.template.ping_mode=A
# node1 (host1)
worker.node1.reference=worker.template
worker.node1.host=<IP_ADDRESS_H1>
worker.node1.redirect=node2
# remote (host2)
worker.node2.reference=worker.template
worker.node2.host=<IP_ADDRESS_H2>
worker.node2.redirect=node1
# Load-balancing behaviour for balanced
worker.balanced.type=lb
worker.balanced.balance_workers=node1,node2
worker.balanced.sticky_session=true
# localhost (had this in here so i wouldnt have to keep reverting this file back to work prior to failover setup)
worker.localhost.reference=worker.template
worker.localhost.host=localhost
3. Create uriworkermap.properties. (identical for both H1 and H2):
# Simple worker configuration file
# Mount the
Servlet context to the ajp13 worker
/jmx-console=balanced
/jmx-console/*=balanced
/web-console=balanced
/web-console/*=balanced
/application=balanced
/application/*=balanced
4. Configuring JBoss to work with mod_jk for each clustered JBoss node.
A. Adding "jvmRoute" to server.xml for both H1 & H2
JBOSS_HOME/server/PRJ/deploy/jboss-web.deployer/server.xml
<Engine name="jboss.web" defaultHost="localhost" jvmRoute="node#"> (where # is "1" for H1 and "2" for H2)
B. For SingleSignOn make sure the "authenticator.SingleSignOn" is disabled (which does not support SSO across a cluster)
and enable SingleSignOn "ClusteredSingleSignOn":
<Valve className="org.jboss.web.tomcat.tc5.sso.ClusteredSingleSignOn"/>
5. For each node in the cluster tell each JBoss
Tomcat instance in the cluster, we need to tell it that mod_jk is in use:
JBOSS_HOME/server/PRJ/deploy/jboss-web.deployer/META-INF/jboss-service.xml
<attribute name="UseJK">true</attribute>
6. For each node in the cluster replicate session data across the nodes in the cluster.
NOTE: Our current Jboss4.0.5 server configuration is based off the default which is why you have to copy over the tc5-cluster sar from the "all" config.
Copy the $JBOSS_HOME/server/all/deploy/tc5-cluster.sar to deploy directory.
Note: From a stock jboss-service.xml do the following:
a. Disable <attribute name="UseRegionBasedMarshalling">false</attribute>
b. Add <attribute name="UseMarshalling">false</attribute> above InactiveOnStartup
c. Remove BuddyReplicationConfig.
d. Disable UDP Cluster in favor of TCP Clustering. Use this version:
<!-- Alternate TCP stack: customize it for your environment, change bind_addr and initial_hosts -->
<config>
<TCP bind_addr="thishostIP" start_port="7810" loopback="true"
tcp_nodelay="true"
recv_buf_size="20000000"
send_buf_size="640000"
discard_incompatible_packets="true"
enable_bundling="true"
max_bundle_size="64000"
max_bundle_timeout="30"
use_incoming_packet_handler="true"
use_outgoing_packet_handler="false"
down_thread="false" up_thread="false"
use_send_queues="false"
sock_conn_timeout="300"
skip_suspected_members="true"/>
<TCPPING initial_hosts="thishostIP[7810],otherhostIP[7810]" port_range="3" timeout="3000"
down_thread="false" up_thread="false"
num_initial_members="3"/>
<MERGE2 max_interval="100000"
down_thread="false" up_thread="false" min_interval="20000"/>
<FD_SOCK down_thread="false" up_thread="false"/>
<FD timeout="10000" max_tries="5" down_thread="false" up_thread="false" shun="true"/>
<VERIFY_SUSPECT timeout="1500" down_thread="false" up_thread="false"/>
<pbcast.NAKACK max_xmit_size="60000"
use_mcast_xmit="false" gc_lag="0"
retransmit_timeout="300,600,1200,2400,4800"
down_thread="false" up_thread="false"
discard_delivered_msgs="true"/>
<pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
down_thread="false" up_thread="false"
max_bytes="400000"/>
<pbcast.GMS print_local_addr="true" join_timeout="3000"
down_thread="false" up_thread="false"
join_retry_timeout="2000" shun="true"
view_bundling="true"/>
<FC max_credits="2000000" down_thread="false" up_thread="false"
min_threshold="0.10"/>
<FRAG2 frag_size="60000" down_thread="false" up_thread="false"/>
<pbcast.STATE_TRANSFER down_thread="false" up_thread="false" use_flush="false"/>
</config>
</attribute>
7. For each node in the cluster Enabling session replication in your application
a. Add distributable tag in the web.xml descriptor.
<distributable/>
b. Define what triggers a session replication. Using "SET_AND_GET":
/usr/local/jboss/server/PRJ/deploy/jmx-console.war/WEB-INF/jboss-web.xml
<jboss-web>
<!-- Uncomment the security-domain to enable security. You will
need to edit the htmladaptor login configuration to setup the
login modules used to authentication users.
<security-domain>
java:/jaas/jmx-console</security-domain>
-->
<!-- The downside of SET_AND_GET is that it can have significant performance
implications, if more than two nodes are used, since even reading immutable objects from the session
(e.g., strings, numbers) will mark the read attributes as needing to be replicated.
-->
<replication-config>
<replication-trigger>SET_AND_GET</replication-trigger>
<replication-granularity>SESSION</replication-granularity>
</replication-config>
</jboss-web>
6. For each node in the cluster set parameters in the Jboss startup script:
Note: If you use custom start/stop scripts for jboss then only edit run.conf in the following manner
-Dbind.address=THISHOSTIP
-Djboss.partition.name=MYPartition
-Djboss.partition.udpGroup=MulticastIP
7. For each node in the cluster update jboss-cache.jar with one in all server config.
8. Logs and Debugging
Can you add this to conf/jboss-log4j.xml to get additional JGroups logging:
<category name="org.jgroups">
<priority value="INFO"/>
</category>
To get JBoss to include additional debug logging for SSO, add this to conf/log4j.xml:
<category name="org.apache.catalina.core.ContainerBase">
<priority value="DEBUG"></priority>
</category>
9. Tests
a.
Test to see if network is correctly configured to UDP multicast:
1. On one node, run the following command, replacing X.X.X.X with that node\'s IP address and YYY.Y.Y.Y with multi-cast IP.
This instance will receive multicast packets.
java -cp jgroups-all.jar org.jgroups.tests.McastReceiverTest -mcast_addr YYY.Y.Y.Y -port 5555 -bind_addr X.X.X.X
2. On the other node, run the following command, replacing X.X.X.X with that node\'s IP address.
This instance will send multicast packets.
java -cp jgroups-all.jar org.jgroups.tests.McastSenderTest -mcast_addr YYY.Y.Y.Y -port 5555 -bind_addr X.X.X.X
3. On the instance sending packets, you can enter text followed by pressing enter.
You should see what you entered echoed on the JVM instance receiving packets.
b. Test for multicast traffic. Put the following text in a file called, "config.xml" in $JBOSS_HOME on both
nodes and run the following:
Note: Changing the frag size to 1000 can account for a common issue with some networks dropping packets over a certain
size.
Command for node1:
java -Djava.net.preferIPv4Stack=true -cp server/eol3/lib/jgroups.jar:lib/commons-logging.jar:lib/concurrent.jar org.jgroups.tests.LargeState -props ./config.xml -provider -size 10000000 -Dbind.address=node2IP
Command for node2:
java -Djava.net.preferIPv4Stack=true -cp server/eol3/lib/jgroups.jar:lib/commons-logging.jar:lib/concurrent.jar org.jgroups.tests.LargeState -props ./config.xml -Dbind.address=node1Ip
config.xml
<config>
<UDP mcast_addr="${jboss.partition.udpGroup:YYY.Y.Y.Y}"
mcast_port="45577"
ucast_recv_buf_size="20000000"
ucast_send_buf_size="640000"
mcast_recv_buf_size="25000000"
mcast_send_buf_size="640000"
loopback="false"
max_bundle_size="64000"
max_bundle_timeout="30"
use_incoming_packet_handler="true"
use_outgoing_packet_handler="true"
ip_ttl="${jgroups.mcast.ip_ttl:2}"
down_thread="false" up_thread="false"
enable_bundling="true"/>
<PING timeout="2000"
down_thread="false" up_thread="false" num_initial_members="3"/>
<MERGE2 max_interval="100000"
down_thread="false" up_thread="false" min_interval="20000"/>
<FD_SOCK down_thread="false" up_thread="false"/>
<FD shun="true" up_thread="false" down_thread="false"
timeout="20000" max_tries="5"/>
<VERIFY_SUSPECT timeout="1500"
up_thread="false" down_thread="false"/>
<pbcast.NAKACK max_xmit_size="60000"
use_mcast_xmit="false" gc_lag="50"
retransmit_timeout="300,600,1200,2400,4800"
down_thread="false" up_thread="false"
discard_delivered_msgs="true"/>
<UNICAST timeout="300,600,1200,2400,3600"
down_thread="false" up_thread="false"/>
<pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
down_thread="false" up_thread="false"
max_bytes="400000"/>
<pbcast.GMS print_local_addr="true" join_timeout="3000"
down_thread="false" up_thread="false"
join_retry_timeout="2000" shun="true"/>
<FC max_credits="2000000" down_thread="false" up_thread="false"
min_threshold="0.10"/>
<FRAG2 frag_size="60000" down_thread="false" up_thread="false"/>
<pbcast.STATE_TRANSFER down_thread="false" up_thread="false"/>
</config>
c. Test for TCP configuration. Put the following text in a file called, "config.xml" in $JBOSS_HOME on both
nodes and run the following:
Command for node1:
/usr/local/java/bin/java -Djava.net.preferIPv4Stack=true -cp jgroups.jar:lib/commons-logging.jar:lib/concurrent.jar org.jgroups.tests.LargeState -props ./config.xml -provider -size 10000000
Command for node2:
/usr/local/java/bin/java -Djava.net.preferIPv4Stack=true -cp jgroups.jar:lib/commons-logging.jar:lib/concurrent.jar org.jgroups.tests.LargeState -props ./config.xml
config.xml:
<config>
<TCP bind_addr="THISHOSTIP" start_port="7810" loopback="true"
tcp_nodelay="true"
recv_buf_size="20000000"
send_buf_size="640000"
discard_incompatible_packets="true"
enable_bundling="true"
max_bundle_size="64000"
max_bundle_timeout="30"
use_incoming_packet_handler="true"
use_outgoing_packet_handler="false"
down_thread="false" up_thread="false"
use_send_queues="false"
sock_conn_timeout="300"
skip_suspected_members="true"/>
<TCPPING initial_hosts="THISHOSTIP[7810],OTHERHOSTIP[7810]" port_range="3"
timeout="3000"
down_thread="false" up_thread="false"
num_initial_members="3"/>
<MERGE2 max_interval="100000"
down_thread="false" up_thread="false" min_interval="20000"/>
<FD_SOCK down_thread="false" up_thread="false"/>
<FD timeout="10000" max_tries="5" down_thread="false" up_thread="false" shun="true"/>
<VERIFY_SUSPECT timeout="1500" down_thread="false" up_thread="false"/>
<pbcast.NAKACK max_xmit_size="60000"
use_mcast_xmit="false" gc_lag="0"
retransmit_timeout="300,600,1200,2400,4800"
down_thread="false" up_thread="false"
discard_delivered_msgs="true"/>
<pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
down_thread="false" up_thread="false"
max_bytes="400000"/>
<pbcast.GMS print_local_addr="true" join_timeout="3000"
down_thread="false" up_thread="false"
join_retry_timeout="2000" shun="true"
view_bundling="true"/>
<FC max_credits="2000000" down_thread="false" up_thread="false"
min_threshold="0.10"/>
<FRAG2 frag_size="60000" down_thread="false" up_thread="false"/>
<pbcast.STATE_TRANSFER down_thread="false" up_thread="false" use_flush="false"/>
</config>