Merge pull request #544 from SnabbCo/next

next: Collected changes for the v2015.08 release

.gitignore

@@ -14,4 +14,4 @@ obj
 /src/syscall.lua
 /src/syscall
 /deps/*.vsn
-/src/program/programs.inc
+/src/programs.inc

Makefile

@@ -16,15 +16,11 @@ all: $(LUAJIT) $(SYSCALL) $(PFLUA)
 install: all
 	install -D src/snabb ${PREFIX}/usr/local/bin/snabb
 
-install_db_node:
-	install -D src/scripts/sysv/init.d/snabb-nfv-sync-master ${PREFIX}/etc/init.d/snabb-nfv-sync-master
-	install -D src/scripts/sysv/default/snabb-nfv-sync-master ${PREFIX}/etc/default/snabb-nfv-sync-master
-
-install_compute_node: install
-	install -D src/scripts/sysv/init.d/snabb-nfv-sync-agent ${PREFIX}/etc/init.d/snabb-nfv-sync-agent
-	install -D src/scripts/sysv/default/snabb-nfv-sync-agent ${PREFIX}/etc/default/snabb-nfv-sync-agent
-
-$(LUAJIT): check_luajit
+$(LUAJIT):
+	@if [ ! -f deps/luajit/Makefile ]; then \
+	    echo "Initializing LuaJIT submodule.."; \
+	    git submodule update --init deps/luajit; \
+	fi
 	@echo 'Building LuaJIT'
 	@(cd deps/luajit && \
 	 $(MAKE) PREFIX=`pwd`/usr/local \
@@ -33,24 +29,19 @@ $(LUAJIT): check_luajit
          git describe > ../luajit.vsn)
 	(cd deps/luajit/usr/local/bin; ln -fs luajit-2.1.0-alpha luajit)
 
-check_luajit:
-	@if [ ! -f deps/luajit/Makefile ]; then \
-	    echo "Initializing LuaJIT submodule.."; \
-	    git submodule update --init deps/luajit; \
-	fi
-
-$(PFLUA): check_pflua
-#       pflua has no tags at time of writing, so use raw commit id
-	@(cd deps/pflua && git rev-parse HEAD > ../pflua.vsn)
-
-check_pflua:
+$(PFLUA): $(LUAJIT)
 	@if [ ! -f deps/pflua/src/pf.lua ]; then \
 	    echo "Initializing pflua submodule.."; \
 	    git submodule update --init deps/pflua; \
 	fi
+#       pflua has no tags at time of writing, so use raw commit id
+	@(cd deps/pflua && git rev-parse HEAD > ../pflua.vsn)
 
-
-$(SYSCALL): check_syscall
+$(SYSCALL): $(PFLUA)
+	@if [ ! -f deps/ljsyscall/syscall.lua ]; then \
+	    echo "Initializing ljsyscall submodule.."; \
+	    git submodule update --init deps/ljsyscall; \
+	fi
 	@echo 'Copying ljsyscall components'
 	@mkdir -p src/syscall/linux
 	@cp -p deps/ljsyscall/syscall.lua   src/
@@ -60,14 +51,9 @@ $(SYSCALL): check_syscall
 	@cp -pr deps/ljsyscall/syscall/shared      src/syscall/
 	@(cd deps/ljsyscall; git describe > ../ljsyscall.vsn)
 
-check_syscall:
-	@if [ ! -f deps/ljsyscall/syscall.lua ]; then \
-	    echo "Initializing ljsyscall submodule.."; \
-	    git submodule update --init deps/ljsyscall; \
-	fi
-
 clean:
 	(cd deps/luajit && $(MAKE) clean)
 	(cd src; $(MAKE) clean; rm -rf syscall.lua syscall)
+	(rm deps/*.vsn)
 
 .SERIAL: all

authorized_keys

@@ -85,3 +85,5 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAzg8e0cW4io7PVp+ID/f8wHldye9eMXv3wwp1TRpo
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqb4RqdSdDPM6A20l2aXjG4v9K99ogsr8lpBGHHl7V2Ek00mrS3bS4sCbvp3YfUFNMZfz+8Fbbl/KIJJngISAw4QTcLMPoUSVfm0H23neJ+LeJjfC6WFlF1orXuTothBjhXvGmEUnpdjfMdA+qNDGNRaVxKBrlCQR1j3JtBzPKBSk/4tAKotOfNmiQlzWPCU7Ea7f96m4I5kimgnuUcFQvhSR+t37JYMeTPu/e3j/nuzVf8dk0yeCHtVvw9CiT7lhho1B/9bM999xDR4scOSBWna2rLrFECn7Ao2d/PoQ5wZE2DRyfXRtonH053VE15Jpc+LJI7VOWXnFONWym8Bu7
 
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmUqhU3Eg6zPJ8MEKpCcBkYRHXd/FVBxmy0DOj3GD5Hdxd/+cBDk/mfIJOy+r4HG2qQJBls9EvvZzcSnF3mqqURfWUDfKJ+fNZ+0vdeTodmgHiOERO5pUyf4h1YxqbxX+cVfELxpPwkGjtWIsvdt+CKuQ1b1rauKgv33YVkm1VJHCBJk8mh/JN5jsplnyu0nR5sPOJQ8rnDOr3vRGa0nGc+G7S03bAUBTmgjbEsRUOG2TJ1TPmkXoCVlR1I4yWUmgSMiKAO8nOFEwQaOQbLhLsvcYF7T4KzF86BUIG6hkm3qfHLPCmdbOfXfFnp8oNZiBq/S0nrHMLT9BGXjfXdbGv sm101
+
+ssh-dss AAAAB3NzaC1kc3MAAACBAMdlCk1NNT2O+np4uzFWFDHP+zTS8uAC6c0mv2miSgAgJxFyfZpJH+HbOuLALCoyPrQbAPb+yPeXvl7xQwAUd94QW3dsX8B70skaxGQMXJdvEu3iDSnpxdeNMW+Ctl4JDHwNoZ93dCxqUqiF5tIE9ock8r1vEZ4d4Xy/LWe+mneVAAAAFQCZ3YEG7uDAfKRxcIK7v4XJyCknCwAAAIA4l8xAexLrEiheg8w8YYGvTtTV20xDaFObLI0fWFpYM0n6g80xkGoM409/1ne6PPqOydCp6dfNcbqf2vCq2WxffjEetMSE5BNk02JctdafO8wiGVFnQd39I+n70SCU/48s/NX+RqWcRgTlwDzp034ZiclDrmrBGVmz5TAJWXT8BgAAAIBbYv/+kxyNdM0HLiQn6/ShTCqK6gkhumDn3a/SS0nHx3LpdlACX9x49a7VTf4tYqctW6LUkE9ei0cvsHWq2ec6Q00UAypCaTtwUjt7vr7HmwuTKV6XOsLkupnEED5jtRgeEz5fuWPIMH6Xg/GENJ5z7N/6AlaOz3Emu6TQtkdwPw== mwiget@mwiget-mba13

src/Makefile

@@ -8,7 +8,7 @@ E= @echo
 # Defined here to detect version mismatches at build time.
 LUAJIT_VSN    := "v2.0.4-306-gfe56522"
 LJSYSCALL_VSN := "v0.10-65-g7081d97"
-PFLUA_VSN     := "27523953c03745d4c4fbbcf81f676bc774e4329e"
+PFLUA_VSN     := "5e2c56baa0cf1ec471719bac83e2a99c4e2d5495"
 
 TEST_SKIPPED="43"
 
@@ -20,19 +20,20 @@ PFLUASRC = $(shell cd ../deps/pflua/src && \
 	         find . -regex '[^\#]*\.lua' -printf '%P ')
 CSRC   = $(shell find . -regex '[^\#]*\.c' -not -regex './arch/.*' -printf '%P ')
 CHDR   = $(shell find . -regex '[^\#]*\.h' -printf '%P ')
-ASM    = $(shell find . -regex '[^\#]*\.dasc' -printf '%P ')
+ASM    = $(shell find . -regex '[^\#]*\.dasl' -printf '%P ')
 ARCHSRC= $(shell find . -regex '^./arch/[^\#]*\.c' -printf '%P ')
 RMSRC  = $(shell find . -name  README.md.src -printf '%P ')
 # regexp is to include program/foo but not program/foo/bar
 PROGRAM = $(shell find program -regex '^[^/]+/[^/]+' -type d -printf '%P ')
-INCSRC = $(shell find . -regex '[^\#]*\.inc' -printf '%P ')
+# sort to eliminate potential duplicate of programs.inc
+INCSRC = $(sort $(shell find . -regex '[^\#]*\.inc' -printf '%P ') programs.inc)
 
 LUAOBJ := $(patsubst %.lua,obj/%_lua.o,$(LUASRC))
 PFLUAOBJ := $(patsubst %.lua,obj/%_lua.o,$(PFLUASRC))
 COBJ   := $(patsubst %.c,obj/%_c.o,    $(CSRC))
 HOBJ   := $(patsubst %.h,obj/%_h.o,    $(CHDR))
 ARCHOBJ:= $(patsubst %.c,obj/%_c.o,    $(ARCHSRC))
-ASMOBJ := $(patsubst %.dasc,obj/%_dasc.o,   $(ASM))
+ASMOBJ := $(patsubst %.dasl,obj/%_dasl.o,   $(ASM))
 JITOBJS:= $(patsubst %,obj/jit_%.o,$(JITSRC))
 EXTRAOBJS := obj/jit_tprof.o obj/jit_vmprof.o obj/strict.o
 RMOBJS := $(patsubst %.src,%,$(RMSRC))
@@ -71,13 +72,9 @@ snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(INCOBJ) $(LU
 	    echo "pflua:"						>&2; \
 	    echo "  require: $(PFLUA_VSN)"				>&2; \
 	    echo "  found:   $(shell cat ../deps/pflua.vsn) "		>&2; \
-	    echo "Please update and rebuild submodules."		>&2; \
-	    echo ""							>&2; \
-	    echo "Cheat sheet:"						>&2; \
-	    echo "  cd snabbswitch"					>&2; \
-	    echo "  git submodule update"				>&2; \
-	    echo "  make clean"						>&2; \
-	    echo "  make -j"						>&2; \
+	    echo ""                                                     >&2; \
+	    echo "Please update your submodules like this:"             >&2; \
+	    echo "  make submods"                                       >&2; \
 	    echo ""							>&2; \
 	    exit 1; \
 	 fi
@@ -91,6 +88,10 @@ snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(INCOBJ) $(LU
 
 all: $(EXE)
 
+# Rebuild after git submodules are updated.
+submods:
+	(cd ..; git submodule update; make clean; make)
+
 $(EXE): snabb
 	$(E) "PROGRAM   $@"
 	$(Q) install -D snabb $@
@@ -168,10 +169,10 @@ $(HOBJ): obj/%_h.o: %.h Makefile | $(OBJDIR)
 	 echo "]=============]") > $(basename $@).luah
 	$(Q) luajit -bg -n $(subst /,.,$*)_h $(basename $@).luah $@
 
-$(ASMOBJ): obj/%_dasc.o: %.dasc $(CHDR) Makefile | $(OBJDIR)
+$(ASMOBJ): obj/%_dasl.o: %.dasl $(CHDR) Makefile | $(OBJDIR)
 	$(E) "ASM       $@"
-	$(Q) luajit ../deps/luajit/dynasm/dynasm.lua -o $@.gen $<
-	$(Q) gcc $(DEBUG) -Wl,-E -I ../deps/luajit/src -I . -I ../deps/luajit -c -Wall -Werror -x c -o $@ $@.gen
+	$(Q) luajit dynasm.lua -o $@.gen $<
+	$(Q) luajit -bg -n $(subst /,.,$*) $@.gen $@
 
 $(JITOBJS): obj/jit_%.o: ../deps/luajit/src/jit/%.lua $(OBJDIR)
 	$(E) "LUA       $@"
@@ -182,15 +183,15 @@ $(RMOBJS): %: %.src
 	$(E) "MARKDOWN  $@"
 	$(Q) scripts/process-markdown $< > $@
 
-$(INCOBJ): obj/%_inc.o: %.inc Makefile program/programs.inc | $(OBJDIR)
+$(INCOBJ): obj/%_inc.o: %.inc Makefile | $(OBJDIR)
 	$(E) "INC       $@"
 	@(echo -n "return [=============["; \
 	 cat $<; \
 	 echo "]=============]") > $(basename $@).luainc
 	$(Q) luajit -bg -n $(subst /,.,$*)_inc $(basename $@).luainc $@
 
 # Create list of programs that exist
-program/programs.inc:
+programs.inc: program
 	@(for d in program/*/; do basename $$d; done) > $@
 
 FORCE:
@@ -225,7 +226,7 @@ doc/snabbswitch.epub: doc/snabbswitch.md
 	$(E) "PANDOC    $@"
 	$(Q) (cd doc; pandoc --self-contained --css="style.css" -S -s --toc --chapters -o snabbswitch.epub snabbswitch.md)
 
-CLEAN = snabb snabbswitch obj bin doc/snabbswitch.* doc/.images/* testlog deps/*.vsn
+CLEAN = snabb snabbswitch obj bin doc/snabbswitch.* doc/.images/* testlog deps/*.vsn programs.inc
 
 clean:
 	$(E) "RM        $(CLEAN)"

src/README.md

@@ -89,13 +89,6 @@ For example: Move packets from input ports to output ports or to a
 network adapter.
 
 
-— Method **myapp:relink**
-
-*Optional*. React to a changes in input/output links (`app.input` and
-`app.output`). This method is called after a link reconfiguration but
-before the next packets are processed.
-
-
 — Method **myapp:reconfig** *arg*
 
 *Optional*. Reconfigure the app with a new *arg*. If this method is not
@@ -207,7 +200,23 @@ following keys are recognized:
 Returns monotonic time in seconds as a floating point number. Suitable
 for timers.
 
+— Variable **engine.busywait**
+
+If set to true then the engine polls continuously for new packets to
+process. This consumes 100% CPU and makes processing latency less
+vulnerable to kernel scheduling behavior which can cause pauses of
+more than one millisecond.
+
+Default: false
 
+— Variable **engine.Hz**
+
+Frequency at which to poll for new input packets. The default value is
+'false' which means to adjust dynamically up to 100us during low
+traffic. The value can be overridden with a constant integer saying
+how many times per second to poll.
+
+This setting is not used when engine.busywait is true.
 
 ## Link (core.link)
 
@@ -342,13 +351,6 @@ represented by a table with the following keys:
 * `size` -  Size in bytes
 * `used` - Bytes used
 
-— Variable **memory.dma_min_addr**
-
-— Variable **memory.dma_max_addr**
-
-Lowest and highest addresses of valid DMA memory. Useful information for
-creating memory maps. Read-only.
-
 — Variable **memory.huge_page_size**
 
 Size of a single huge page in bytes. Read-only.

src/README.md.src

@@ -34,11 +34,21 @@ implement a specific interface, which is used by the Snabb Switch engine
 to organize an *App Network*.
 
     DIAGRAM: Network
-    +-------+            +-------+
-    |       |            |       |
-    | App 1 *------------* App 2 |
-    |       |   (Link)   |       |
-    +-------+            +-------+
+                   +---------+
+                   |         |
+                +->* Filter0 *--+
+                |  |         |  |
+                |  +---------+  |
+    +---+----+  |               |  +----+---+
+    |        *--+               +->*        |
+    |  NIC0  |                     |  NIC1  |
+    |        *<-+               +--*        |
+    +---+----+  |               |  +----+---+
+                |  +---------+  |
+                |  |         |  |
+                +--* Filter1 *<-+
+                   |         |
+                   +---------+
 
 Usually, a Snabb Switch design will create a series of apps, interconnect
 these in a desired way using *links* and finally pass the resulting app
@@ -216,7 +226,23 @@ following keys are recognized:
 Returns monotonic time in seconds as a floating point number. Suitable
 for timers.
 
+— Variable **engine.busywait**
 
+If set to true then the engine polls continuously for new packets to
+process. This consumes 100% CPU and makes processing latency less
+vulnerable to kernel scheduling behavior which can cause pauses of
+more than one millisecond.
+
+Default: false
+
+— Variable **engine.Hz**
+
+Frequency at which to poll for new input packets. The default value is
+'false' which means to adjust dynamically up to 100us during low
+traffic. The value can be overridden with a constant integer saying
+how many times per second to poll.
+
+This setting is not used when engine.busywait is true.
 
 ## Link (core.link)
 

src/apps/bridge/README.md

@@ -0,0 +1,82 @@
+# Bridge Apps
+
+A `bridge` app implements a basic Ethernet bridge with split-horizon
+semantics. It has an arbitrary number of ports. For each input port there
+must exist an output port with the same name. Each port name is a member
+of at most one *split-horizon group*. If it is not a member of a
+split-horizon group, the port is also called a *free port*. Packets
+arriving on a free input port may be forwarded to all other output
+ports. Packets arriving on an input port that belongs to a split-horizon
+group are never forwarded to any output port belonging to the same
+split-horizon group. There are two `bridge` implementations available:
+`apps.bridge.flooding` and apps.bridge.learning`.
+
+![bridge](.images/bridge.png)
+
+## Configuration
+
+A `bridge` app accepts a table as its configuration argument. The
+following keys are defined:
+
+— Key **ports**
+
+*Optional*. An array of free port names. The default is no free ports.
+
+— Key **split_horizon_groups**
+
+*Optional*. A table mapping split-horizon groups to arrays of port
+names. The default is no split-horizon groups.
+
+— Key **config**
+
+*Optional*. The configuration of the actual bridge implementation.
+
+
+# Flooding bridge (apps.bridge.flooding)
+
+The flooding `bridge` app implements the simplest possible bridge, which
+floods a packet arriving on an input port to all output ports within its
+scope according to the split-horizon topology.
+
+## Configuration
+
+The flooding `bridge` app ignores the *config* key of its configuration.
+
+
+# Learning bridge (apps.bridge.learning)
+
+The learning `bridge` app implements a *learning bridge* using a [Bloom
+filter](https://en.wikipedia.org/wiki/Bloom_filter) to store the set of
+MAC source addresses of packets arriving on each input port. When a
+packet is received it is forwarded to all output ports whose
+corresponding input ports match the packet's destination MAC address.
+When no input port matches, the packet is flooded to all output ports.
+Multicast MAC addresses are always flooded to all output ports associated
+with the input port. The scoping rules according to the split-horizon
+topology apply unchanged.
+
+## Configuration
+
+The learning `bridge` app accepts a table as the value of the *config*
+key of its configuration. The following keys are defined:
+
+— Key **mac_table_size**
+
+**Optional**. Expected maximum number of MAC addresses to store in each
+per-port Bloom filter. Default is 1000.
+
+— Key **fp_rate**
+
+**Optional**. Maximum rate of false-positives for look-ups in the Bloom
+filters, provided the number of distinct objects stored in the filter
+does not exceed *mac_table_size*. Default is 0.001.
+
+— Key **timeout**
+
+**Optional**. Timeout for learned MAC addresses in seconds. Default is
+60.
+
+— Key **verbose**
+
+**Optional**. A boolean value. If true, enables the printing of debugging
+output. Default is `false`.