Merge branch 'next' for release 2015.06 "Kiwi".

.gitignore

@@ -14,3 +14,4 @@ obj
 /src/syscall.lua
 /src/syscall
 /deps/*.vsn
+/src/program/programs.inc

src/Makefile

@@ -191,9 +191,9 @@ $(INCOBJ): obj/%_inc.o: %.inc Makefile program/programs.inc | $(OBJDIR)
 
 # Create list of programs that exist
 program/programs.inc:
-	@ls -1d program/*/ | xargs basename -a > $@
+	@(for d in program/*/; do basename $$d; done) > $@
 
-.FORCE: program/programs.inc
+FORCE:
 
 # extra/ third party bits and pieces
 obj/strict.o: extra/strict.lua | $(OBJDIR)

src/arch/avx2.c

@@ -63,7 +63,7 @@ static inline uint32_t cksum_avx2_loop(unsigned char *p, size_t n)
 
 uint16_t cksum_avx2(unsigned char *p, size_t n, uint32_t initial)
 {
-    uint32_t sum = ntohs(initial);
+    uint32_t sum = initial;
 
     if (n < 128) { return cksum_generic(p, n, initial); }
     if (n >= 64) {

src/arch/sse2.c

@@ -66,7 +66,7 @@ static inline uint32_t cksum_sse2_loop(unsigned char *p, size_t n)
 
 uint16_t cksum_sse2(unsigned char *p, size_t n, uint32_t initial)
 {
-  uint32_t sum = ntohs(initial);
+  uint32_t sum = initial;
 
   if (n < 128) { return cksum_generic(p, n, initial); }
   int unaligned = (unsigned long) p & 0xf;

src/core/main.lua

@@ -59,7 +59,7 @@ end
 
 function usage ()
    print("Usage: "..ffi.string(C.argv[0]).." <program> ...")
-   local programs = require("program.programs_inc"):gsub("[a-z]+", "  %1")
+   local programs = require("program.programs_inc"):gsub("%S+", "  %1")
    print()
    print("This snabb executable has the following programs built in:")
    print(programs)

src/core/snabbswitch.c

@@ -14,6 +14,6 @@ int main(int snabb_argc, char **snabb_argv)
   argv = snabb_argv;
   lua_State* L = luaL_newstate();
   luaL_openlibs(L);
-  return luaL_dostring(L, "require \"core.main\"");
+  return luaL_dostring(L, "require \"core.startup\"");
 }
 

src/core/startup.lua

@@ -0,0 +1,6 @@
+local ok, err = pcall(require, "core.main")
+if not ok then
+   print("startup: unhandled exception")
+   print(err)
+   os.exit(1)
+end

src/doc/branches.md

@@ -11,14 +11,36 @@ The current state of each branch with respect to master is visible here:
 #### master
 
     BRANCH: master git://github.com/lukego/snabbswitch
-    Synchronization point with changes recommended for all users.
+    Stable branch suitable for development and deployment.
 
-    - Makes monthly releases.
-    - Changes are gated by the SnabbBot CI
-    - Currently Linux/x86-64 with GNU tools (more in the future)
+    - Always contains a stable release that is safe to pull from.
+    - Updated monthly with new features and weekly with new bug fixes.
+    - Changes are gated by the SnabbBot CI.
 
     Maintainer: Luke Gorrie <luke@snabb.co> and Max Rottenkolber <max@mr.gy>
 
+#### next
+
+    BRANCH: next git://github.com/lukego/snabbswitch
+    Test and integration branch for new development.
+
+    - Contains the changes for the next monthly feature release.
+    - Merges Pull Requests that pass code review on Github.
+    - Cycles between unstable and stable with the release schedule.
+
+    Maintainer: Luke Gorrie <luke@snabb.co>
+
+#### fixes
+
+    BRANCH: fixes git://github.com/lukego/snabbswitch
+    Test and integration branch for bug fixes.
+
+    - Contains the changes for the next weekly maintenance release.
+    - Merges Pull Requests that fix bugs in the latest release.
+    - Generally stable.
+
+    Maintainer: Luke Gorrie <luke@snabb.co>
+
 #### documenation-fixups
 
     BRANCH: documentation-fixups git://github.com/eugeneia/snabbswitch

src/doc/documentation-guide.mk2

@@ -0,0 +1,202 @@
+# Contributing Documentation in SnabbSwitch
+
+## README.md, README.md.src and README(.inc)
+
+The SnabbSwitch documentation is organized in README's spread across the
+code base. If you commit changes you should look for the "nearest" README
+file related to the modules you changed and update these. There are three
+different kinds of README files:
+
+* `README.md` — A portion of the SnabbSwitch manual, embedded by GitHub
+  too. These are often (but not always) artifacts built from a `.src`
+  file. Edit these if no `.src` is available (see below).
+* `README.md.src` — A build recipe. If available, this is the one you
+  must edit. These are formatted in [GitHub Flavored
+  Markdown](https://help.github.com/articles/github-flavored-markdown/).
+* `README(.inc)` — Plain text files included as the `--help` message of
+  SnabbSwitch programs. These should be using only the ASCII character
+  set to ensure compatibility with older terminals.
+
+For instance if you had changed the API of `lib.protocol.ethernet` you'd
+need to update the documentation in `lib/protocol/README.md.src`. It is
+important that you use the correct header level (e.g. `##`, `###`,
+...). Which level you are at can be seen in `doc/genbook.sh` (see
+*Building A Standalone Documentation*).
+
+Be sure to know who your target audience is. We recognize three different
+groups:
+
+* **Casual users:** Some programs are targeting *anyone* using Snabb
+  Switch, make sure that their `--help` message (`README.inc`) does not
+  require prior knowledge.
+* **Power users:** Public APIs like `core.packet` or
+  `lib.protocol.ethernet` are there for app designers to use. These
+  should be well documented but may require knowledge of Lua and other
+  parts of the Snabb Switch system.
+* **Contributors:** Some documents (like the one you are reading now) are
+  facing towards Snabb Switch contributors. They can be frequently
+  updating and even contain open questions. We should ensure that these
+  always reflect the current state of affairs.
+
+### Building README.md Files
+
+In order to recreate a `README.md` file from its `.src` you need to
+`make` it. E.g. in case of our example above you would run:
+
+```
+make lib/protocol/README.md
+```
+
+You need to commit the resulting `README.md` (and possibly generated
+diagram images, see *Including Diagrams*) alongside the updated
+`README.md.src`.
+
+### Including Diagrams
+
+The main reason for the `README.md.src` to `README.md` step is that we
+use a custom Markdown pre-processor to be able to embed ASCII diagrams in
+our documentation. These ASCII diagrams will be rendered to pretty images
+by [ditaa](http://ditaa.sourceforge.net/). In order to build `README.md`
+files containing diagrams you will need a local installation of ditaa.
+
+In order to use the diagram pre-processor you need to embed a ditaa
+diagram in a block *indented with four space characters* lead by a
+`DIAGRAM: <title>` header:
+
+```
+Normal paragraph...
+
+    DIAGRAM: My Diagram
+    +------+
+    |A Box |<--With an arrow
+    +------+
+```
+
+## Building A Standalone Documentation
+
+In order to build a complete SnabbSwitch reference manual you can use
+`make doc/snabbswitch.html`, `make doc/snabbswitch.epub` and `make
+doc/snabbswitch.pdf` to build the HTML, ePUB and PDF versions
+respectively. Alternatively, `make book` can be used to build all three
+at once. For these tasks to work you will need a local installation of
+[Pandoc](http://johnmacfarlane.net/pandoc/).
+
+On Ubuntu you can install everything required to produce HTML, PDF and
+epub versions with the following `apt-get` command:
+
+```
+sudo apt-get install pandoc pandoc-citeproc texlive-latex-recommended texlive-xetex texlive-luatex texlive-fonts-recommended texlive-latex-extra texlive-fonts-extra
+```
+
+You can change the organization and contents of the resulting file by
+editing `doc/genbook.sh`, which is really just a primitive shell script
+that concatenates the various `README.md` files we have.
+
+# Stylistic Conventions
+
+## Anatomy Of A Module Section
+
+Every module has its own subsection in the SnabbSwitch manual, and all
+these sections start off the same way:
+
+```
+### Protocol Header (lib.protocol.header)
+
+The `lib.protocol.header` module contains stuff...
+```
+
+That is: The header contains the title of the module as well as its
+*module path* in parentheses. The header is followed by a paragraph that
+again names the module path and summarizes the module's purpose. This
+introduction can be as detailed as the module required. Some modules are
+obvious, some deserve along-form high-level introduction with examples.
+
+If the module in question is an App, the introduction must be followed by
+a diagram visually describing its inputs and outputs. E.g.:
+
+```
+    DIAGRAM: MyApp
+              +------------+
+              |            |
+      in ---->*    MyApp   *----> out
+              |            |
+              +------------+
+```
+
+After the introduction follows a complete description of every *external*
+symbol of the module. External means symbols that are part of the modules
+public API. Every symbol gets its own special mention of the form:
+
+```
+— <Type> **<qualified.name>** <type-specific>
+
+Paragraphs describing the symbol...
+```
+
+The `—` character is an *em dash*. Currently we use the following types:
+Variable, Function, Method and Key. Variable and function names are
+prefixed with their module name (separated from the symbol name by a
+`.`). Methods are prefixed with their class name (separated from the
+symbol name by a `:`). Functions and methods are followed by their
+parameter lists. E.g.:
+
+```
+— Variable **module.some_constant**
+
+— Function **module.some_function** *arg1*, *arg2*
+
+— Method **class:some_method** *arg1*, *arg2*
+```
+
+If the module in question is an App, the symbol definitions must be
+followed by a sub-section "Configuration" that elaborates on the App's
+configuration parameter. E.g.
+
+```
+### Configuration
+
+The `nd_light` app accepts a table as its configuration argument. The
+following keys are defined:
+
+— Key **local_mac**
+
+*Required*. Local MAC address as a string or in binary representation.
+
+— Key **retrans**
+
+*Optional*. Number of neighbor solicitation retransmissions. Default is
+unlimited retransmissions.
+```
+
+Each key's description must be preceded by either `*Required*` or
+`*Optional*` to signify if its a required or an optional parameter. Each
+key's description must also declare the expected type of the argument
+value. Each optional key's description must end in a sentence defining
+its default value.
+
+
+## Markup Conventions
+
+We markup source code literals in `code` font. E.g.: "The `foobar` module
+is nice" and "`mod.fun(bla)` will make your dreams come true". Parameter
+identifiers are marked up in *italic* font. E.g.: "`mod.foo` takes an
+argument *bar*".
+
+UNIX system calls should be mentioned like so: `usleep(3)`.
+
+We markup specific *concepts* we introduce in italic font the first time
+they are mentioned in order to signify to the reader that a specific
+concept has a well defined meaning.
+
+## Terminology And Normalized Language
+
+The parameter names used in method and function description do not need
+to reflect the names used in the source code. Instead use long,
+descriptive names made out of full words when sensible.
+
+Symbol definition are written in third person, e.g.: "Returns a number"
+instead of "Return a number". When describing default behavior we say
+"The default is..." instead of "Defaults to..." etc.
+
+When in doubt, turn to the [Lua Reference Manual](http://www.lua.org/manual/5.1/)
+for linguistic and stylistic reference.

src/doc/git-workflow.md

@@ -0,0 +1,86 @@
+# Snabb Switch Git Workflow
+
+This document explains the Git workflows that we use to develop and
+release Snabb Switch.
+
+## Overview
+
+Snabb Switch development follows a few well-known patterns:
+
+- We follow the distributed development model [described by Linus
+  Torvald](https://www.youtube.com/watch?v=4XpnKHJAok8) and pioneered
+  by the Linux kernel.
+- We use a [merge based workflow](https://www.atlassian.com/git/articles/git-team-workflows-merge-or-rebase/).
+- We use the [fork and pull](https://help.github.com/articles/using-pull-requests/#fork--pull)
+  model of collaboration.
+- We aim to support ad-hoc collaboration inspired by the
+  [DMZ Flow](https://gist.github.com/djspiewak/9f2f91085607a4859a66).
+
+## HOWTO
+
+### Download and update the latest release
+
+1. Clone the [SnabbCo/snabbswitch](https://github.com/SnabbCo/snabbswitch) repository.
+2. Check out and build the `master` branch.
+3. Pull when you want to update to the latest stable release.
+
+### Develop and contribute an improvement
+
+1. [Create your own fork](https://help.github.com/articles/fork-a-repo/) of Snabb Switch on Github.
+2. Develop and debug your contribution on a new [topic branch](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows#Topic-Branches) based on the latest `master`.
+3. Make a final cleanup of your code before review. (Last chance to rebase.)
+4. Submit a Github [Pull Request](https://help.github.com/articles/using-pull-requests/#initiating-the-pull-request)
+   to the `master` branch.
+5. Respond to feedback and correct problems by pushing additional commits.
+
+There are two milestones in the process of accepting your change:
+
+1. Your change is merged onto a branch that feeds `master`, for
+   example `next`, `fixes`, `documentation-fixes`, or `nfv`. From this
+   point the owner of that branch will push your work upstream
+   together with other related changes. They might ask you for help
+   but otherwise your work is done.
+2. Your change is merged onto `master`. This could happen in a series
+   of merge steps, for example `nfv->next->master`. Once this happens
+   your code has been officially released as part of Snabb Switch.
+
+### Develop and maintain a new program
+
+Snabb Switch includes programs like `snabbnfv`, `packetblaster`, and
+`snsh`. Here is how you can create a new program and take charge of
+its development.
+
+1. [Fork](https://help.github.com/articles/fork-a-repo/) your own
+   repository on Github.
+2. Create a [long-lived branch](branches.md) where new development of your program will be done.
+3. Create a directory `src/program/myapplication/` and develop your program.
+4. `git merge master` regularly to stay synchronized with the main line of development.
+5. Optional: Send releases of your application to `master` with Pull Requests.
+
+The code in your `src/program/myapplication/` directory is developed
+according to your own rules and tastes. If there are parts of this
+code that you especially want to have reviewed (or do not want to have
+reviewed) then please explain this in your Pull Request. The only
+necessary review is to make sure that programs do not negatively
+impact each other or change shared code without enough review.
+
+Pull Requests that make changes to your application will be referred
+to you for merge onto your branch.
+
+Use the *Develop and contribute an improvement* workflow to make
+changes to the core Snabb Switch code. Please do not bundle
+substantial changes to the core software with updates to your program.
+
+If you do not want to include your program in the main Snabb Switch
+release then this is no problem. You can simply pull from `master` to
+receive updates and skip the step of pushing back.
+
+### To help maintain Snabb Switch
+
+Here are the best ways to help maintain Snabb Switch:
+
+1. Review Pull Requests to help people quickly improve them.
+2. Test the `next` branch and help fix problems before releases.
+3. Contribute new `selftest` cases to make our CI more effective.
+4. Maintain a [branch](branches.md) where you accept Pull Requests and push them upstream.
+