Freebsd 11.0 – FEMP

FEMP

This is a basic setup of FEMP, it’s splitted in several parts. First you have to select either installing it from ports or building it all from source files, both works but the port install takes less time todo. After you have done the installation move forward to configuration and trying it out. This is by no mean a optimal guid, but a fast jump into installing FEMP.

Through Ports

###################################################################
#
# First i start off with Nginx since its the easy part of the install
#
###################################################################

# Enter the ports folder for nginx
cd /usr/ports/www/nginx

# Configure the build, Basic settings are fine for beginners
make config-recursive

# Install and clean up
make install clean

# Add nginx to startup upon boot of the os
echo 'nginx_enable="YES"' >> /etc/rc.conf

# Create a www data folder that is easy to access for the future. And change owner.
mkdir /home/www
mkdir /home/www/default
chown www:www /home/www


###################################################################
#
#  Next step is install the database, i prefer mariadb.
#
###################################################################

# Enter the ports folder for mariadb server 10.1
cd /usr/ports/databases/mariadb101-server/

# Configure the build, accept basic settings
make config-recursive

# Install and clean up, this will take a while.
make install clean

# Setup the password for mysql root account, default is none if asked for passwd.
# Set a strong password, and then default settings on the rest.
/usr/local/bin/mysql_secure_installation

# Add it to startup upon boot of os.
echo 'mysql_enable="YES"' >> /etc/rc.conf

###################################################################
#
# Now time for the last part install php.
#
###################################################################

# Enter the ports folder for mariadb server 10.1
cd /usr/ports/lang/php70

# Ensure that fpm is marked.
make config-recursive

# Install and clean-up
make install clean

# Add php to spawn upon boot
echo 'php_fpm_enable="YES"' >> /etc/rc.conf

# Copy the production file into php.ini
cp /usr/local/etc/php.ini-production /usr/local/etc/php.ini

# install following extensions Json, session, mysqli, mbstring, 
# gd, openssl, zlib, zip, pdo_mysql
cd /usr/ports/lang/php70-extensions
make config-recursive
make install clean
Building from source
######################################
#  Fetch all the files
######################################
cd /tmp/
fetch ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.40.tar.gz
fetch http://www.cpan.org/src/5.0/perl-5.24.1.tar.gz
fetch http://nginx.org/download/nginx-1.11.9.tar.gz
fetch http://se2.php.net/get/php-7.1.1.tar.gz/from/this/mirror
fetch http://acc.dl.osdn.jp/php-i18n/52624/libmbfl-1.2.0.tar.gz
fetch http://www.zlib.net/zlib-1.2.11.tar.gz
fetch --no-verify-peer https://www.openssl.org/source/openssl-1.0.2k.tar.gz
fetch http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz
fetch --no-verify-peer https://curl.haxx.se/download/curl-7.52.1.tar.gz
fetch --no-verify-peer https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/bash/bash-4.4.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/bash/readline-5.1.tar.gz
fetch ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz
fetch --no-verify-peer https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz
fetch ftp://ftp.gnu.org/gnu/gss/gss-1.0.3.tar.gz
fetch http://ftp.ddg.lth.se/mariadb//mariadb-10.1.21/source/mariadb-10.1.21.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/automake/automake-1.15.tar.gz
fetch --no-verify-peer https://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
fetch http://ftp.acc.umu.se/mirror/gnu.org/gnu/libtool/libtool-2.4.tar.gz
fetch ftp://xmlsoft.org/libxml2/libxml2-2.9.4.tar.gz
fetch http://downloads.webmproject.org/releases/webp/libwebp-0.6.0.tar.gz
fetch --no-verify-peer https://kent.dl.sourceforge.net/project/libpng/libpng16/1.6.28/libpng-1.6.28.tar.gz
fetch --no-verify-peer https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
fetch http://www.ijg.org/files/jpegsrc.v9b.tar.gz
fetch --no-verify-peer https://mirrors.netix.net/sourceforge/m/mc/mcrypt/Libmcrypt/2.5.8/libmcrypt-2.5.8.tar.gz



######################################
#  Un-tar everything in /tmp
######################################
tar -zxf pcre-8.40.tar.gz | tar -zxf nginx-1.11.9.tar.gz | tar -zxf mirror
tar -zxf mariadb-10.1.21.tar.gz  | tar -zxf perl-5.24.1.tar.gz | tar -zxf zlib-1.2.11.tar.gz
tar -zxf libmbfl-1.2.0.tar.gz | tar -zxf openssl-1.0.2k.tar.gz
tar -zxf libarchive-3.2.2.tar.gz | tar -zxf curl-7.52.1.tar.gz | tar -zxf cmake-3.7.2.tar.gz
tar -zxf m4-1.4.18.tar.gz | tar -zxf bison-3.0.4.tar.gz | tar -zxf bash-4.4.tar.gz
tar -zxf readline-5.1.tar.gz | tar -zxf libffi-3.2.1.tar.gz | tar -zxf Python-2.7.13.tgz
tar -zxf gss-1.0.3.tar.gz | tar -zxf mariadb-10.1.21.tar.gz | tar -zxf automake-1.15.tar.gz
tar -zxf autoconf-2.69.tar.gz | tar -zxf libtool-2.4.tar.gz | tar -zxf libmcrypt-2.5.8.tar.gz
tar -zxf libxml2-2.9.4.tar.gz | tar -zxf libwebp-0.6.0.tar.gz
tar -zxf libpng-1.6.28.tar.gz | tar -zxf gmp-6.1.2.tar.xz | tar -zxf jpegsrc.v9b.tar.gz


######################################
#
#  Required files/packages  before
#  install of php,nginx,mariadb
#
######################################
cd /tmp/perl-5.24.1
./configure.gnu
make
make install

cd /tmp/openssl-1.0.2k
./config -fPIC shared
make
make install

cd /tmp/libarchive-3.2.2
./configure
make
make install

cd /tmp/curl-7.52.1
./configure
make
make install

cd /tmp/cmake-3.7.2
./configure
make
make install

cd /tmp/m4-1.4.18
./configure
make
make install

cd /tmp/bison-3.0.4
./configure
make
make install

cd /tmp/bash-4.4
./configure
make
make install

cd /tmp/readline-5.1
./configure
make
make install

cd /tmp/libffi-3.2.1
./configure
make
make install

cd /tmp/Python-2.7.13
./configure --enable-shared --enable-optimizations
make
make install

cd /tmp/gss-1.0.3
./configure
make
make install

cd /tmp/pcre-8.40
./configure
make
make install

cd /tmp/autoconf-2.69
./configure
make
make install

cd /tmp/automake-1.15
./configure
make
make install

cd /tmp/libtool-2.4
./configure
make
make install

cd /tmp/libmbfl-1.2.0
chmod +x buildconf
./buildconf
./configure
make
make install

cd /tmp/libxml2-2.9.4
./configure
make
make install

cd /tmp/libwebp-0.6.0
./configure
make
make install

cd /tmp/libpng-1.6.28
./configure
make
make install

cd /tmp/gmp-6.1.2
./configure
make
make install

cd /tmp/jpeg-9b
./configure
make
make install

cd /tmp/libmcrypt-2.5.8
./configure --disable-posix-threads
make
make install

######################################
#  MARIADB
######################################
cd /tmp/mariadb-10.1.21
pw groupadd mysql
pw adduser mysql -g mysql -d /usr/local/mysql

# In the menu just press c for configure, takes a while.
# Write no on 'JEMALLOC_STATIC_LIBRARY' & 'WITH_JEMALLO',
# and last line to write no on 'PLUGIN_TOKUDB'
# press c and then g to save config & quit.
ccmake .

# Compiling using 4 threads.
make -j4

# Install everything
make install

# Add the run file
mkdir /usr/local/etc/
mkdir /usr/local/etc/rc.d
cd /usr/local/etc/rc.d
fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/2017/mysql-server
chmod +x /usr/local/etc/rc.d/mysql-server

# Add the default my.cnf (change this later on..)
mkdir /var/db/mysql
cd /var/db/mysql/
fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/2017/my.cnf

# Start sql server
/usr/local/etc/rc.d/mysql-server onestart

# Set password for root mysql user
/usr/local/mysql/bin/mysql_secure_installation


######################################
#  NGINX
######################################
cd /tmp/nginx-1.11.9
./configure --sbin-path=/usr/local/sbin/nginx --conf-path=/usr/local/etc/nginx/nginx.conf --pid-path=/var/run/nginx.pid --with-cc-opt="-I /usr/local/include" --with-ld-opt="-L /usr/local/lib" --with-http_stub_status_module
make
make install

# Add the run file
cd /usr/local/etc/rc.d
fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/nginx
chmod +x /usr/local/etc/rc.d/nginx

mkdir /var/log/nginx/
mkdir /home/www
mkdir /home/www/default
chown www:www /home/www

######################################
#  PHP
######################################
cd /tmp/php-7.1.1

# Compiling php with the needed extensions.
./configure \
--enable-fpm \
--with-fpm-user=www \
--with-fpm-group=www \
--enable-libxml \
--enable-zip \
--with-bz2=shared \
--with-curl=shared \
--with-gd \
--with-jpeg-dir=/usr \
--with-png-dir=/usr \
--with-webp-dir=/usr \
--enable-gd-native-ttf \
--with-gmp=shared \
--enable-mbstring \
--enable-bcmath \
--with-mcrypt \
--with-mhash=shared \
--with-mysqli=/usr/local/mysql/bin/mysql_config \
--with-pdo-mysql \
--enable-sockets \
--with-zlib \
--enable-ftp \
--enable-sysvmsg \
--enable-sysvsem \
--enable-sysvshm \
--with-openssl

# Install
make -j4
make install

# Add the run file
cd /usr/local/etc/rc.d
fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/2017/php-fpm
chmod +x /usr/local/etc/rc.d/php-fpm

# Copy the production ini to the correct folder and so php loads the correct ini
cd /usr/local/etc/
fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/2017/php.ini
chmod 755 php.ini

cp /usr/local/etc/php-fpm.d/www.conf.default /usr/local/etc/php-fpm.d/www.conf
cp /usr/local/etc/php-fpm.conf.default /usr/local/etc/php-fpm.conf

# Change this line in /usr/local/etc/php-fpm.conf
include=NONE/etc/php-fpm.d/*.conf
# To
include=/usr/local/etc/php-fpm.d/*.conf

# And in php-fpm.conf uncomment and change
;pid = run/php-fpm.pid
# To
pid = /var/run/php-fpm.pid


######################################
#  Last part + cleanup
######################################
# Add programs to boot, nginx,php,mysql.
echo 'php_fpm_enable="YES"' >> /etc/rc.conf
echo 'mysql_enable="YES"' >> /etc/rc.conf
echo 'nginx_enable="YES"' >> /etc/rc.conf

# Remove all files in /tmp
rm -R /tmp/*
######################################
#  Now continue to configuration before trying out the services!
######################################


Configurations

There are a couple of basic configurations that need to be done. Lets start with configuring php-fpm, and continue with php.ini and then get nginx to work with php.

/usr/local/etc/php-fpm.d/www.conf

# Next we uncomment the user and group of the ownership
;listen.owner = www
;listen.group = www
;listen.mode = 0660

# To following 
listen.owner = www
listen.group = www
listen.mode = 0660

/usr/local/etc/php.ini

# Security option that needs to be uncommented and set to 0
# ;cgi.fix_pathinfo=1

# Change it to
cgi.fix_pathinfo=0

# If you plan to transfer large files, change this to preferred value
upload_max_filesize = 2M

# And if you have larger transfers make sure that the execution time is extended
max_execution_time = 30

/usr/local/etc/nginx/nginx.conf
First delete the old nginx.conf and replace it with the one bellow.

# Set the worker process to auto, or the amount of cpus/core you have
worker_processes  auto;
# Store errors in logs/error.log and log even warnings
error_log /var/log/nginx/error.log warn;
# Create and store pid in logs/pid
pid /var/run/nginx.pid;

events {
    worker_connections  1024;
    multi_accept on;
    use kqueue;
}


http {
    # Define the mime.types
    include       mime.types;
    default_type  application/octet-stream;

    # How logs are structured
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    # Store access logs in logs/access.log
    access_log  /var/log/nginx/access.log  main;

    # Optimising nginx, when serving static content.
    sendfile        on;
    tcp_nopush     on;
    tcp_nodelay    on;
    # Set keepalive time out to 15 sec
    keepalive_timeout  15;

    # Enable gzip compression
    gzip  on;
    
    # Disable gzip on old crap iexplorer
    gzip_disable "MSIE [1-6]\.(?!.*SV1)";

    # Include site configurations from the sites/ folder.
    include  /usr/local/etc/nginx/sites/*;
}

/usr/local/etc/nginx/sites/default.site

server {
    listen 80;
    server_name localhost;
    root /home/www/default;

    location / {
        index  index.php index.html index.htm;
    }

    location = /50x.html {
        root   html;
    }

    location ~ \.php$ {
        try_files $uri =404;
        fastcgi_split_path_info ^(.+\.php)(/.+)$;
        fastcgi_pass 127.0.0.1:9000;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include        fastcgi_params;
    }

    location ~ /\.ht$ {
         deny  all;
    }
}
/usr/local/etc/my.cnf

If you want to change some settings to mariadb, the settings are located in /usr/local/share/mysql/ with filnames such my-small.cnf, my-medium.cnf, my-larg.cnf, my-huge.cnf open the preferred size, configure it and save it as /usr/local/etc/my.cnf

And we are done! Good luck optimising!

Testing the build

Basic test could be done by running a phpinfo() call in the index file just to make sure it works
/home/www/default/index.php

< ?php
phpinfo();
? >

Or we could do a quick test to see if php, mariadb and nginx works together, by first trying to install and use phpmyadmin & wordpress.

Install Phpmyadmin

# Enter the tmp folder
cd /tmp

# Fetch the phpmyadmin
fetch --no-verify-peer https://files.phpmyadmin.net/phpMyAdmin/4.6.5.2/phpMyAdmin-4.6.5.2-all-languages.tar.gz

# Untar the file
tar -zxf phpMyAdmin-4.6.5.2-all-languages.tar.gz

# Make directory phpmyadmin in webfolder
mkdir /home/www/default/phpmyadmin

# Copy the files to the webfolder
cp -R /tmp/phpMyAdmin-4.6.5.2-all-languages/* /home/www/default/phpmyadmin

#remove temp files
rm -R /tmp/phpMyAdmin-4.6.5.2-all-languages | rm phpMyAdmin-4.6.5.2-all-languages.tar.gz

Now you should be able to access phpmyadmin @ http://ip_to_server/phpmyadmin While trying it out, ensure that you make a database for the wordpress install.

Install WP

# Enter the tmp folder
cd /tmp

# Fetch the latest wordpress file
fetch --no-verify-peer https://wordpress.org/latest.tar.gz

# Untar the file
tar -zxf latest.tar.gz

# Copy wordpress file into default www folder 
cp -R /tmp/wordpress/* /home/www/default/

# Remove temp files
rm -R /tmp/wordpress | rm latest.tar.gz 

# Done

After adding the wordpress files to the www-folder you have to enter http://ip_to_server/ and follow the installation guide, if everything is setup correctly the setup should work flawlessly. Remember to set the right folder permissions to get it to work, correct permissions can be found at: https://codex.wordpress.org/Changing_File_Permissions

There are alot of improvements that could be done to this setup. For example adding caching features, or adding/removing extensions, and solving the snmp issue if the service provider blocks it and so on.

Freebsd 11.0 – LanCache

During one of the lan events I was hosting for my students I noticed the total bandwidth for 3 days, it was very high. In this case it was 2,15TB in total of data transferred in just a couple of days. No one complaint because the event was hosted on a 500/500Mbit fiber connection, but if I were to extend the event with more than 35 users problem would arise.

First though was a squid proxy, but since a lot of the data was from steam or similar game services I started to look into something called LAN-Caching. Which should cache as much as possible. I ended up with the configuration bellow, I will use that in the future to reduce bandwidth usage on the wan interface. And this configuration will be used on 2 identical high-perfomance servers connected to a core Cisco switch.

There are some improvements that I still have todo with this setup,I’m still doing try runs with some specific users to learn which service fails and what I fail to cache. This Isn’t a final build and must always be optimised and changed before an event.

Stats from the first event:

Network drawing – Practical example

Requirments
Complete hdd/ssd setup & mount it as /data. 
Complete the network structure Gateway + switches(read about lagg).
Fresh install of unbound or selected it during freebsd 11.0 install.
First install nginx – Server 1 & 2
# Select basic settings + add rewrite module + ssl + slice
cd /usr/ports/www/nginx
make config-recursive
make install clean
echo 'nginx_enable="YES"' >> /etc/rc.conf
/boot/loader.conf – Server 1 & 2
# Load carp module
carp_load="YES"
# Buffer incoming connections until certain http request arrives 
accf_http_load="YES"
# Wait for data accept filter
accf_data_load="YES"
# Load lagg module
if_lagg_load="YES"
/etc/sysctl.conf – Server 1 & 2
# Carp settings
net.inet.carp.allow=1
net.inet.carp.preempt=1
net.inet.carp.log=1
# Allow ip forwarding
net.inet.ip.forwarding=1
/etc/rc.conf – Server 1
# LAGG & Carp Server 1
ifconfig_bce0="up"
ifconfig_bce1="up"
defaultrouter="10.0.2.1"
cloned_interfaces="lagg0"
ifconfig_lagg0="laggproto lacp laggport bce0 laggport bce1 10.0.2.2/24 up"

ifconfig_lagg0_aliases="\
        inet vhid 1 advskew 100 pass paswd alias 10.0.2.4/32 \
        inet vhid 2 advskew 100 pass paswd alias 10.0.2.5/32 \
        inet vhid 3 advskew 200 pass paswd alias 10.0.2.6/32 \
        inet vhid 4 advskew 100 pass paswd alias 10.0.2.7/32 \
        inet vhid 5 advskew 200 pass paswd alias 10.0.2.8/32 \
        inet vhid 6 advskew 100 pass paswd alias 10.0.2.9/32 \
        inet vhid 7 advskew 200 pass paswd alias 10.0.2.10/32 \
        inet vhid 8 advskew 100 pass paswd alias 10.0.2.11/32 \
        inet vhid 9 advskew 200 pass paswd alias 10.0.2.12/32 \
        inet vhid 10 advskew 100 pass paswd alias 10.0.2.13/32 \
        inet vhid 11 advskew 200 pass paswd alias 10.0.2.14/32 \
        inet vhid 12 advskew 100 pass paswd alias 10.0.2.15/32"
/etc/rc.conf – Server 2
# LAGG & Carp Server 2
ifconfig_bce0="up"
ifconfig_bce1="up"
defaultrouter="10.0.2.1"
cloned_interfaces="lagg0"
ifconfig_lagg0="laggproto lacp laggport bce0 laggport bce1 10.0.2.3/24 up"

ifconfig_lagg0_aliases="\
        inet vhid 1 advskew 200 pass paswd alias 10.0.2.4/32 \
        inet vhid 2 advskew 200 pass paswd alias 10.0.2.5/32 \
        inet vhid 3 advskew 100 pass paswd alias 10.0.2.6/32 \
        inet vhid 4 advskew 200 pass paswd alias 10.0.2.7/32 \
        inet vhid 5 advskew 100 pass paswd alias 10.0.2.8/32 \
        inet vhid 6 advskew 200 pass paswd alias 10.0.2.9/32 \
        inet vhid 7 advskew 100 pass paswd alias 10.0.2.10/32 \
        inet vhid 8 advskew 200 pass paswd alias 10.0.2.11/32 \
        inet vhid 9 advskew 100 pass paswd alias 10.0.2.12/32 \
        inet vhid 10 advskew 200 pass paswd alias 10.0.2.13/32 \
        inet vhid 11 advskew 100 pass paswd alias 10.0.2.14/32 \
        inet vhid 12 advskew 200 pass paswd alias 10.0.2.15/32"
/etc/hosts – Server 1 & 2
# Link some host names to specific ip's, can be used in unbound and nginx
10.0.2.5        lancache-steam
10.0.2.6        lancache-riot
10.0.2.7        lancache-blizzard
10.0.2.8        lancache-hirez
10.0.2.9        lancache-origin
10.0.2.10       lancache-sony
10.0.2.11       lancache-arenanetworks
10.0.2.12       lancache-ubisoft
10.0.2.13       lancache-gog
10.0.2.14       lancache-turbine
10.0.2.15       lancache-microsoft
/etc/unbound/unbound.conf – Server 1 & 2
# Basic settings
server:
        interface:10.0.2.2
        interface:0.0.0.0
        interface:10.0.2.4
        access-control: 0.0.0.0/0 allow
        private-address: 10.0.2.0/24
        ip-transparent: yes
        do-ip4: yes
        do-udp: yes
        do-tcp:yes
        do-daemonize:yes
        username: unbound
        directory: /var/unbound
        chroot: /var/unbound
        pidfile: /var/run/local_unbound.pid
        auto-trust-anchor-file: /var/unbound/root.key

include: /var/unbound/lancaching.conf
include: /var/unbound/forward.conf
include: /var/unbound/lan-zones.conf
include: /var/unbound/control.conf
include: /var/unbound/conf.d/*.conf

Fetching lancaching.conf – Server 1 & 2
# Enter the unbound folder, fetch the file: lancaching.conf
cd /var/unbound/
fetch --no-verify-peer http://lyxi.ga/wp-content/uploads/2017/lancaching.conf
Nginx cache folders + logs – Server 1 & 2
# Create folders for logs and cache data
mkdir /data/ | mkdir /data/www/
mkdir /data/www/logs/ | mkdir /data/www/cache/
mkdir /data/www/cache/tmp | mkdir /data/www/cache/other | mkdir /data/www/cache/installs

# Change owner of the folder and set full permission on /data
chown -R www:www /data | chmod -R 777 /data

# Download the nginx configs for lan-cache, remove old nginx.conf & unpack.
rm /usr/local/etc/nginx/nginx.conf
cd /tmp/
# This contains a modified version of junkhacker's lancache,to work on freebsd.
fetch --no-verify-peer  http://lyxi.ga/wp-content/uploads/2017/lancachemaster.tar.gz
tar -zxf lancachemaster.tar.gz
cp -R /tmp/lancachemaster/* /usr/local/etc/nginx/
rm -R /tmp/lancachemaster | rm lancachemaster.tar.gz

# Original source of configs bellow for latest update: 
https://github.com/junkhacker/lancache

# Rebooting nginx 
/usr/local/etc/rc.d/nginx restart
Try it out
# Good to use command to monitor current connection speed 
systat -ifstat

# Once everything is setup and default dns is set to 10.0.2.4 you should be able to
# launch steam, do a fresh download, remove the game from library and download again.
# Result should be something similar as the photos bellow.

# This is by no mean optimal, modifications need to be done a couple of days before the event.

Photo before files cached:

Photo after files were cached:

Sources

https://blog.multiplay.co.uk/2014/04/lancache-dynamically-caching-game-installs-at-lans-using-nginx/
https://github.com/ForayJones/lancache
https://github.com/junkhacker/lancache
https://github.com/bntjah/lancache
https://blog.yolocation.pro/index.php/2016/02/03/how-to-install-lancache-on-debian/

Freebsd 11.0 – Link Aggregation (Lagg)

In this short post i’ll go through what link aggregation is, how it works and when/where it could be applied. And in the end a basic practical example how to setup lagg in freebsd 11.0

This is link aggregation

Link aggregation is pretty much grouping two more more network interfaces into one network group, creating either failover or a high throughput connection. The more interfaces you add to the group the higher the throughput you get and lesser risk of the connection to fail. There are also some weaknesses in link aggregation, for example if by co-incident two larger file transfers decides to go on link 1 in the group, then it’s only able to max the speed on link 1, and link 2 in the group is unused.

There are also different ways to use link aggregation which you could read at freebsd’s documentation under: round robin, lacp, failover, and fec / loadbalance.

Where / when to use it
Some use it to cut cost’s, for example its much cheaper to activate link aggregation on existing hardware and get some extra speed than buying a whole new equipment with better link speeds. New equipment usually means more work hours for the workers, planning, and paying for the equipment. But i would never recommend doing this, because you are just pushing the issue another couple of months forward. Instead its a solution to prevent downtime in the wait for new equipment to arrive, and give time to plan ahead and implement the new solution / upgrade. Link aggregation could also be used for redundancy one link breaks the other one takes the load, one other time to use link aggregation is for events that require more speed than the usual load, for example conventions or LANS.

Practical exampel

Easy setup to enable link aggregation on two network ports. Both ports will listen to the ip 10.0.2.2, remember to enable link aggregation on the switch or the network device connected to this server.

/boot/loader.conf

# load lagg upon boot
if_lagg_load="YES"

/etc/rc.conf

# Start both network cards
ifconfig_em0="up"
ifconfig_em1="up"

# Clone them into lagg0 and a loopback lo1
cloned_interfaces="lagg0 lo1"

# Put the lagg interface up, and select the lagg protocol lacp, 
# group the interfaces (laggports), and set a ip address to lagg0 10.0.2.2 
ifconfig_lagg0="up laggproto lacp laggport em0 laggport em1 10.0.2.2/24"

# set the default route (gateway)
defaultrouter="10.0.2.1"

Source
Much of the information is from the good documentation that freebsd has to offer, in this case: https://www.freebsd.org/doc/handbook/network-aggregation.html

Freebsd 11.0 – Carp

In this short article i’ll go through CARP(Common Address Redundancy Protocol), how it works and when/where it could be applied. And in the end a basic practical example how to activate and configure CARP on freebsd 11.0.

This is CARP

Carp is a network protocol which allows multiple hosts / servers on the same network to share a set of ip addresses thus creating virtual addresses. This is to prevent downtime by creating a failover redundancy group, but it could also provide load balancing functionality. Carp could be compared to other protocols such as VRRP and HSRP.

It’s like having two office buildings next to each other sharing the same entrance. That way you have one master door leading to the master office, and a backup door leading backup office, but they are sharing the same address. If master door & office is blocked by some reason or out of service, the backup door/office becomes the master till the moment the other door is unblocked or works again. That way a visitor a.k.a client will be able to use the service without noticing anything different.

TL;DR one virtual / shared ip address on several hosts, one server goes down the backup takes over, thus failover but ip only level.

Practical exampel

The example bellow will show how to setup two server with CARP, each server will have their own static ip and a shared virtual ip. And we also have a password to prevent abuse. The setup will be as the picture bellow. And one usage area could be as in this example: proxy interface, if one proxy server goes down the other one continues, thus redundancy.

Master Working

Backup line taking over

Server 1 – /etc/rc.conf
# setup interface em0 with ip 10.0.2.2 and start it.
ifconfig_em0="inet 10.0.2.2 netmask 255.255.255.0"

# Clone interface and name it carp0
cloned_interfaces="carp0"

# set default route to gw in this case 10.0.2.1
defaultrouter="10.0.2.1"

# skew must be =<100
# setup virtual iface on interface carp0, virtual host id 1, advskew 100(master)
# shared password passwd, shared Virtual ip 10.0.2.5
ifconfig_carp0_proxy="inet vhid 1 advskew 100 pass paswd alias 10.0.2.5/32"
Server 2 - /etc/rc.conf
# setup interface em0 with ip 10.0.2.2 and start it.
ifconfig_em0="inet 10.0.2.3 netmask 255.255.255.0"

# Clone interface and name it carp0
cloned_interfaces="carp0"

# set default route to gw in this case 10.0.2.1
defaultrouter="10.0.2.1"

# skew must be =<100
# setup virtual iface on interface em0, virtual host id 1, advskew 200(slave/backup) 
# shared password passwd, shared Virtual ip 10.0.2.5
ifconfig_carp0_proxy="inet vhid 1 advskew 200 pass paswd alias 10.0.2.5/32"
Server 1 & 2 - /boot/loader.conf
# load carp upon boot
carp_load="YES"
Server 1 & 2 - /etc/sysctl.conf
# Allow carp data on the interfaces (enabled by default, but just to make sure)
net.inet.carp.allow=1
# Unsure how to explain but, servers will look ahead and make sure master is master
# and backup is a backup.
net.inet.carp.preempt=1
# Enable logging of the vhid's, logging bad carp data. 
net.inet.carp.log=1
Testing

Could be done by allowing a client to ping 10.0.2.5, and shutdown server 1. Or setup a web-server on both servers with the same configuration on both servers and visit http://10.0.2.5 and do some basic tests to ensure that it works.

Source

And for those who wants more information about carp, look at the good freebsd documentation bellow
https://www.freebsd.org/doc/handbook/carp.html

Freebsd 11.0 – NginX

A quick installation of NginX with a standard configuration. This is by no mean optimal. NginX is a lightweight web-server, which also could be used for loadbalancing, cache server, and it offers many nice features that could be used to provide different services.

Portsnap installation of NginX:
# Keep portsnap up to date before installing.
portsnap fetch update

#Enter the NginX portsnap folder.
cd /usr/ports/www/nginx

# Compilation config for NginX and the same for the required packages.
# Accept default settings unless you have read the manual and you are sure
# what modules to add to the build.
make config-recursive

# Install everything / also compiles all extra features selected and clean up
make install clean

# start nginx at boot
echo 'nginx_enable="YES"' >> /etc/rc.conf
Latest version just by compiling it on your own:
# Enter the tmp folder
cd /tmp

# Replace the x.y.z with the latest stable version
# First we have to install pcre, won't work with pcre2.
# (ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/)
fetch ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-x.y.tar.gz

# Un-tar the file
tar -zxf pcre-x.y.tar.gz

# Setup the conf for the compile
./configure

# Compile
make

# Install the files
make install

# Delete temp files
cd /tmp | rm -R /tmp/pcre-x.y/ | rm /tmp/pcre-x.y.tar.gz

# Download the latest version of Nginx Directly from the site (http://nginx.org/en/download.html)
fetch http://nginx.org/download/nginx-x.y.z.tar.gz

# Un-tar the file
tar -zxf nginx-x.y.z.tar.gz

# Enter the folder
cd nginx-x.y.z

# Configure before the compile (options here: http://nginx.org/en/docs/configure.html).
./configure --sbin-path=/usr/local/sbin/nginx --conf-path=/usr/local/etc/nginx/nginx.conf --pid-path=/var/run/nginx.pid --with-cc-opt="-I /usr/local/include" --with-ld-opt="-L /usr/local/lib" --with-http_stub_status_module

# Compile
make

# Copy the files into the correct folder and so on.
make install

# Remove the un-needed files, this will auto delete if you have installed 
# freebsd 11.0 with the security option 'empty temp upon reboot'
cd /tmp | rm -R /tmp/nginx-x.y.z/ | rm /tmp/nginx-x.y.z.tar.gz

# Create this directory if it doesn't exist
mkdir /usr/local/etc/rc.d

#Download the nginx boot script and chmod it with +x
cd /usr/local/etc/rc.d | fetch --no-verify-peer https://lyxi.ga/wp-content/uploads/nginx | chmod +x /usr/local/etc/rc.d/nginx

#Adding NginX into rc.conf to get it to start upon boot.
echo 'nginx_enable="YES"' &gt;&gt; /etc/rc.conf
Files and hints
# Configuration files located in: /usr/local/etc/nginx/
# Bin: /usr/local/sbin/nginx
# Pid file: /var/run/nginx.pid
# Logs for debugging located in: /usr/local/nginx/logs

# Start, stop, restart, & checkconfiguration
/usr/local/etc/rc.d/nginx start
/usr/local/etc/rc.d/nginx stop
/usr/local/etc/rc.d/nginx restart
/usr/local/etc/rc.d/nginx checkconfig
Sources worth reading

https://www.nginx.com/resources/admin-guide/installing-nginx-open-source/
http://nginx.org/en/docs/configure.html
http://nginx.org/en/docs/beginners_guide.html