Freebsd 11.0 – LanCache

During one of the lan events I was hosting for my students I noticed the total bandwidth for 3 days, it was very high. In this case it was 2,15TB in total of data transferred in just a couple of days. No one complaint because the event was hosted on a 500/500Mbit fiber connection, but if I were to extend the event with more than 35 users problem would arise.

First though was a squid proxy, but since a lot of the data was from steam or similar game services I started to look into something called LAN-Caching. Which should cache as much as possible. I ended up with the configuration bellow, I will use that in the future to reduce bandwidth usage on the wan interface. And this configuration will be used on 2 identical high-perfomance servers connected to a core Cisco switch.

There are some improvements that I still have todo with this setup,I’m still doing try runs with some specific users to learn which service fails and what I fail to cache. This Isn’t a final build and must always be optimised and changed before an event.

Stats from the first event:

Network drawing – Practical example

Requirments
Complete hdd/ssd setup & mount it as /data. 
Complete the network structure Gateway + switches(read about lagg).
Fresh install of unbound or selected it during freebsd 11.0 install.
First install nginx – Server 1 & 2
# Select basic settings + add rewrite module + ssl + slice
cd /usr/ports/www/nginx
make config-recursive
make install clean
echo 'nginx_enable="YES"' >> /etc/rc.conf
/boot/loader.conf – Server 1 & 2
# Load carp module
carp_load="YES"
# Buffer incoming connections until certain http request arrives 
accf_http_load="YES"
# Wait for data accept filter
accf_data_load="YES"
# Load lagg module
if_lagg_load="YES"
/etc/sysctl.conf – Server 1 & 2
# Carp settings
net.inet.carp.allow=1
net.inet.carp.preempt=1
net.inet.carp.log=1
# Allow ip forwarding
net.inet.ip.forwarding=1
/etc/rc.conf – Server 1
# LAGG & Carp Server 1
ifconfig_bce0="up"
ifconfig_bce1="up"
defaultrouter="10.0.2.1"
cloned_interfaces="lagg0"
ifconfig_lagg0="laggproto lacp laggport bce0 laggport bce1 10.0.2.2/24 up"

ifconfig_lagg0_aliases="\
        inet vhid 1 advskew 100 pass paswd alias 10.0.2.4/32 \
        inet vhid 2 advskew 100 pass paswd alias 10.0.2.5/32 \
        inet vhid 3 advskew 200 pass paswd alias 10.0.2.6/32 \
        inet vhid 4 advskew 100 pass paswd alias 10.0.2.7/32 \
        inet vhid 5 advskew 200 pass paswd alias 10.0.2.8/32 \
        inet vhid 6 advskew 100 pass paswd alias 10.0.2.9/32 \
        inet vhid 7 advskew 200 pass paswd alias 10.0.2.10/32 \
        inet vhid 8 advskew 100 pass paswd alias 10.0.2.11/32 \
        inet vhid 9 advskew 200 pass paswd alias 10.0.2.12/32 \
        inet vhid 10 advskew 100 pass paswd alias 10.0.2.13/32 \
        inet vhid 11 advskew 200 pass paswd alias 10.0.2.14/32 \
        inet vhid 12 advskew 100 pass paswd alias 10.0.2.15/32"
/etc/rc.conf – Server 2
# LAGG & Carp Server 2
ifconfig_bce0="up"
ifconfig_bce1="up"
defaultrouter="10.0.2.1"
cloned_interfaces="lagg0"
ifconfig_lagg0="laggproto lacp laggport bce0 laggport bce1 10.0.2.3/24 up"

ifconfig_lagg0_aliases="\
        inet vhid 1 advskew 200 pass paswd alias 10.0.2.4/32 \
        inet vhid 2 advskew 200 pass paswd alias 10.0.2.5/32 \
        inet vhid 3 advskew 100 pass paswd alias 10.0.2.6/32 \
        inet vhid 4 advskew 200 pass paswd alias 10.0.2.7/32 \
        inet vhid 5 advskew 100 pass paswd alias 10.0.2.8/32 \
        inet vhid 6 advskew 200 pass paswd alias 10.0.2.9/32 \
        inet vhid 7 advskew 100 pass paswd alias 10.0.2.10/32 \
        inet vhid 8 advskew 200 pass paswd alias 10.0.2.11/32 \
        inet vhid 9 advskew 100 pass paswd alias 10.0.2.12/32 \
        inet vhid 10 advskew 200 pass paswd alias 10.0.2.13/32 \
        inet vhid 11 advskew 100 pass paswd alias 10.0.2.14/32 \
        inet vhid 12 advskew 200 pass paswd alias 10.0.2.15/32"
/etc/hosts – Server 1 & 2
# Link some host names to specific ip's, can be used in unbound and nginx
10.0.2.5        lancache-steam
10.0.2.6        lancache-riot
10.0.2.7        lancache-blizzard
10.0.2.8        lancache-hirez
10.0.2.9        lancache-origin
10.0.2.10       lancache-sony
10.0.2.11       lancache-arenanetworks
10.0.2.12       lancache-ubisoft
10.0.2.13       lancache-gog
10.0.2.14       lancache-turbine
10.0.2.15       lancache-microsoft
/etc/unbound/unbound.conf – Server 1 & 2
# Basic settings
server:
        interface:10.0.2.2
        interface:0.0.0.0
        interface:10.0.2.4
        access-control: 0.0.0.0/0 allow
        private-address: 10.0.2.0/24
        ip-transparent: yes
        do-ip4: yes
        do-udp: yes
        do-tcp:yes
        do-daemonize:yes
        username: unbound
        directory: /var/unbound
        chroot: /var/unbound
        pidfile: /var/run/local_unbound.pid
        auto-trust-anchor-file: /var/unbound/root.key

include: /var/unbound/lancaching.conf
include: /var/unbound/forward.conf
include: /var/unbound/lan-zones.conf
include: /var/unbound/control.conf
include: /var/unbound/conf.d/*.conf

Fetching lancaching.conf – Server 1 & 2
# Enter the unbound folder, fetch the file: lancaching.conf
cd /var/unbound/
fetch --no-verify-peer http://lyxi.ga/wp-content/uploads/2017/lancaching.conf
Nginx cache folders + logs – Server 1 & 2
# Create folders for logs and cache data
mkdir /data/ | mkdir /data/www/
mkdir /data/www/logs/ | mkdir /data/www/cache/
mkdir /data/www/cache/tmp | mkdir /data/www/cache/other | mkdir /data/www/cache/installs

# Change owner of the folder and set full permission on /data
chown -R www:www /data | chmod -R 777 /data

# Download the nginx configs for lan-cache, remove old nginx.conf & unpack.
rm /usr/local/etc/nginx/nginx.conf
cd /tmp/
# This contains a modified version of junkhacker's lancache,to work on freebsd.
fetch --no-verify-peer  http://lyxi.ga/wp-content/uploads/2017/lancachemaster.tar.gz
tar -zxf lancachemaster.tar.gz
cp -R /tmp/lancachemaster/* /usr/local/etc/nginx/
rm -R /tmp/lancachemaster | rm lancachemaster.tar.gz

# Original source of configs bellow for latest update: 
https://github.com/junkhacker/lancache

# Rebooting nginx 
/usr/local/etc/rc.d/nginx restart
Try it out
# Good to use command to monitor current connection speed 
systat -ifstat

# Once everything is setup and default dns is set to 10.0.2.4 you should be able to
# launch steam, do a fresh download, remove the game from library and download again.
# Result should be something similar as the photos bellow.

# This is by no mean optimal, modifications need to be done a couple of days before the event.

Photo before files cached:

Photo after files were cached:

Sources

https://blog.multiplay.co.uk/2014/04/lancache-dynamically-caching-game-installs-at-lans-using-nginx/
https://github.com/ForayJones/lancache
https://github.com/junkhacker/lancache
https://github.com/bntjah/lancache
https://blog.yolocation.pro/index.php/2016/02/03/how-to-install-lancache-on-debian/

Freebsd 11.0 – Link Aggregation (Lagg)

In this short post i’ll go through what link aggregation is, how it works and when/where it could be applied. And in the end a basic practical example how to setup lagg in freebsd 11.0

This is link aggregation

Link aggregation is pretty much grouping two more more network interfaces into one network group, creating either failover or a high throughput connection. The more interfaces you add to the group the higher the throughput you get and lesser risk of the connection to fail. There are also some weaknesses in link aggregation, for example if by co-incident two larger file transfers decides to go on link 1 in the group, then it’s only able to max the speed on link 1, and link 2 in the group is unused.

There are also different ways to use link aggregation which you could read at freebsd’s documentation under: round robin, lacp, failover, and fec / loadbalance.

Where / when to use it
Some use it to cut cost’s, for example its much cheaper to activate link aggregation on existing hardware and get some extra speed than buying a whole new equipment with better link speeds. New equipment usually means more work hours for the workers, planning, and paying for the equipment. But i would never recommend doing this, because you are just pushing the issue another couple of months forward. Instead its a solution to prevent downtime in the wait for new equipment to arrive, and give time to plan ahead and implement the new solution / upgrade. Link aggregation could also be used for redundancy one link breaks the other one takes the load, one other time to use link aggregation is for events that require more speed than the usual load, for example conventions or LANS.

Practical exampel

Easy setup to enable link aggregation on two network ports. Both ports will listen to the ip 10.0.2.2, remember to enable link aggregation on the switch or the network device connected to this server.

/boot/loader.conf

# load lagg upon boot
if_lagg_load="YES"

/etc/rc.conf

# Start both network cards
ifconfig_em0="up"
ifconfig_em1="up"

# Clone them into lagg0 and a loopback lo1
cloned_interfaces="lagg0 lo1"

# Put the lagg interface up, and select the lagg protocol lacp, 
# group the interfaces (laggports), and set a ip address to lagg0 10.0.2.2 
ifconfig_lagg0="up laggproto lacp laggport em0 laggport em1 10.0.2.2/24"

# set the default route (gateway)
defaultrouter="10.0.2.1"

Source
Much of the information is from the good documentation that freebsd has to offer, in this case: https://www.freebsd.org/doc/handbook/network-aggregation.html

Freebsd 11.0 – Carp

In this short article i’ll go through CARP(Common Address Redundancy Protocol), how it works and when/where it could be applied. And in the end a basic practical example how to activate and configure CARP on freebsd 11.0.

This is CARP

Carp is a network protocol which allows multiple hosts / servers on the same network to share a set of ip addresses thus creating virtual addresses. This is to prevent downtime by creating a failover redundancy group, but it could also provide load balancing functionality. Carp could be compared to other protocols such as VRRP and HSRP.

It’s like having two office buildings next to each other sharing the same entrance. That way you have one master door leading to the master office, and a backup door leading backup office, but they are sharing the same address. If master door & office is blocked by some reason or out of service, the backup door/office becomes the master till the moment the other door is unblocked or works again. That way a visitor a.k.a client will be able to use the service without noticing anything different.

TL;DR one virtual / shared ip address on several hosts, one server goes down the backup takes over, thus failover but ip only level.

Practical exampel

The example bellow will show how to setup two server with CARP, each server will have their own static ip and a shared virtual ip. And we also have a password to prevent abuse. The setup will be as the picture bellow. And one usage area could be as in this example: proxy interface, if one proxy server goes down the other one continues, thus redundancy.

Master Working

Backup line taking over

Server 1 – /etc/rc.conf
# setup interface em0 with ip 10.0.2.2 and start it.
ifconfig_em0="inet 10.0.2.2 netmask 255.255.255.0"

# Clone interface and name it carp0
cloned_interfaces="carp0"

# set default route to gw in this case 10.0.2.1
defaultrouter="10.0.2.1"

# skew must be =<100
# setup virtual iface on interface carp0, virtual host id 1, advskew 100(master)
# shared password passwd, shared Virtual ip 10.0.2.5
ifconfig_carp0_proxy="inet vhid 1 advskew 100 pass paswd alias 10.0.2.5/32"
Server 2 - /etc/rc.conf
# setup interface em0 with ip 10.0.2.2 and start it.
ifconfig_em0="inet 10.0.2.3 netmask 255.255.255.0"

# Clone interface and name it carp0
cloned_interfaces="carp0"

# set default route to gw in this case 10.0.2.1
defaultrouter="10.0.2.1"

# skew must be =<100
# setup virtual iface on interface em0, virtual host id 1, advskew 200(slave/backup) 
# shared password passwd, shared Virtual ip 10.0.2.5
ifconfig_carp0_proxy="inet vhid 1 advskew 200 pass paswd alias 10.0.2.5/32"
Server 1 & 2 - /boot/loader.conf
# load carp upon boot
carp_load="YES"
Server 1 & 2 - /etc/sysctl.conf
# Allow carp data on the interfaces (enabled by default, but just to make sure)
net.inet.carp.allow=1
# Unsure how to explain but, servers will look ahead and make sure master is master
# and backup is a backup.
net.inet.carp.preempt=1
# Enable logging of the vhid's, logging bad carp data. 
net.inet.carp.log=1
Testing

Could be done by allowing a client to ping 10.0.2.5, and shutdown server 1. Or setup a web-server on both servers with the same configuration on both servers and visit http://10.0.2.5 and do some basic tests to ensure that it works.

Source

And for those who wants more information about carp, look at the good freebsd documentation bellow
https://www.freebsd.org/doc/handbook/carp.html

Freebsd 11.0 – NginX

A quick installation of NginX with a standard configuration. This is by no mean optimal. NginX is a lightweight web-server, which also could be used for loadbalancing, cache server, and it offers many nice features that could be used to provide different services.

Portsnap installation of NginX:
# Keep portsnap up to date before installing.
portsnap fetch update

#Enter the NginX portsnap folder.
cd /usr/ports/www/nginx

# Compilation config for NginX and the same for the required packages.
# Accept default settings unless you have read the manual and you are sure
# what modules to add to the build.
make config-recursive

# Install everything / also compiles all extra features selected and clean up
make install clean

# start nginx at boot
echo 'nginx_enable="YES"' >> /etc/rc.conf
Latest version just by compiling it on your own:
# Enter the tmp folder
cd /tmp

# Replace the x.y.z with the latest stable version
# First we have to install pcre, won't work with pcre2.
# (ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/)
fetch ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-x.y.tar.gz

# Un-tar the file
tar -zxf pcre-x.y.tar.gz

# Setup the conf for the compile
./configure

# Compile
make

# Install the files
make install

# Delete temp files
cd /tmp | rm -R /tmp/pcre-x.y/ | rm /tmp/pcre-x.y.tar.gz

# Download the latest version of Nginx Directly from the site (http://nginx.org/en/download.html)
fetch http://nginx.org/download/nginx-x.y.z.tar.gz

# Un-tar the file
tar -zxf nginx-x.y.z.tar.gz

# Enter the folder
cd nginx-x.y.z

# Configure before the compile (options here: http://nginx.org/en/docs/configure.html).
./configure --sbin-path=/usr/local/sbin/nginx --conf-path=/usr/local/etc/nginx/nginx.conf --pid-path=/var/run/nginx.pid --with-cc-opt="-I /usr/local/include" --with-ld-opt="-L /usr/local/lib" --with-http_stub_status_module

# Compile
make

# Copy the files into the correct folder and so on.
make install

# Remove the un-needed files, this will auto delete if you have installed 
# freebsd 11.0 with the security option 'empty temp upon reboot'
cd /tmp | rm -R /tmp/nginx-x.y.z/ | rm /tmp/nginx-x.y.z.tar.gz

# Create this directory if it doesn't exist
mkdir /usr/local/etc/rc.d

#Download the nginx boot script and chmod it with +x
cd /usr/local/etc/rc.d | fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/nginx | chmod +x /usr/local/etc/rc.d/nginx

#Adding NginX into rc.conf to get it to start upon boot.
echo 'nginx_enable="YES"' &gt;&gt; /etc/rc.conf
Files and hints
# Configuration files located in: /usr/local/etc/nginx/
# Bin: /usr/local/sbin/nginx
# Pid file: /var/run/nginx.pid
# Logs for debugging located in: /usr/local/nginx/logs

# Start, stop, restart, & checkconfiguration
/usr/local/etc/rc.d/nginx start
/usr/local/etc/rc.d/nginx stop
/usr/local/etc/rc.d/nginx restart
/usr/local/etc/rc.d/nginx checkconfig
Sources worth reading

https://www.nginx.com/resources/admin-guide/installing-nginx-open-source/
http://nginx.org/en/docs/configure.html
http://nginx.org/en/docs/beginners_guide.html