Revamp and improve config system

 Now features:
 - keys can be binary blobs (useful for accept and server)
 - values can be unique or accumulators

Signed-off-by: Laurent Bercot <ska@appnovation.com>

8 files changed, 268 insertions, 278 deletions

diff --git a/src/config/conftree.c b/src/config/conftree.c
deleted file mode 100644
index 573646a..0000000
--- a/src/config/conftree.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* ISC license. */
-
-#include <skalibs/genalloc.h>
-#include <skalibs/avltree.h>
-#include <skalibs/cdbmake.h>
-
-#include "shibari-cache-config-internal.h"
-
-static repo conftree = \
-{ \
-  .ga = GENALLOC_ZERO, \
-  .tree = AVLTREE_INIT(8, 3, 8, &node_dtok, &node_cmp, &conftree.ga), \
-  .storage = &g.storage \
-} ;
-
-void confnode_start (node *node, char const *key, size_t filepos, uint32_t line)
-{
-  return node_start(&g.storage, node, key, filepos, line) ;
-}
-
-void confnode_add (node *node, char const *s, size_t len)
-{
-  return node_add(&g.storage, node, s, len) ;
-}
-
-node const *conftree_search (char const *key)
-{
-  return repo_search(&conftree, key) ;
-}
-
-void conftree_add (node const *node)
-{
-  return repo_add(&conftree, node) ;
-}
-
-void conftree_update (node const *node)
-{
-  return repo_update(&conftree, node) ;
-}
-
-static int confnode_write (uint32_t d, unsigned int h, void *data)
-{
-  node *nod = genalloc_s(node, &conftree.ga) + d ;
-  (void)h ;
-  return cdbmake_add((cdbmaker *)data, conftree.storage->s + nod->key, nod->keylen, conftree.storage->s + nod->data, nod->datalen) ;
-}
-
-int conftree_write (cdbmaker *cm)
-{
-  return avltree_iter(&conftree.tree, &confnode_write, cm) ;
-}
diff --git a/src/config/defaults.c b/src/config/defaults.c
index d4ff2a8..9fef25b 100644
--- a/src/config/defaults.c
+++ b/src/config/defaults.c
@@ -1,6 +1,6 @@
 /* ISC license. */
 
-#include <stddef.h>
+#include <stdint.h>
 
 #include "shibari-cache-config-internal.h"
 
@@ -8,19 +8,19 @@ struct defaults_s
 {
   char const *key ;
   char const *value ;
-  size_t vlen ;
+  uint32_t vlen ;
 } ;
 
 #define REC(k, v, n) { .key = (k), .value = (v), .vlen = (n) }
-#define RECS(k, v) REC(k, v, sizeof(v))
+#define RECS(k, v) REC(k, (v), sizeof(v))
 #define RECU32(k, u) { .key = (k), .value = (char const [4]){ (u) >> 24 & 0xffu, (u) >> 16 & 0xffu, (u) >> 8 & 0xffu, (u) & 0xffu }, .vlen = 4 }
 
 static struct defaults_s const defaults[] =
 {
   RECU32("G:logv", 1),
   RECU32("G:maxtcp", 256),
-  REC("G:listen4", "\0\0\0\0\0\35", 6),
-  REC("G:listen6", "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\35", 18),
+  REC("G:listen4", "\177\0\0\1", 4),
+  REC("G:listen6", "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1", 16),
 
   REC("R4:",
    "\0\306\51\0\4"
@@ -58,14 +58,25 @@ static struct defaults_s const defaults[] =
 
 void conf_defaults (void)
 {
-  for (struct defaults_s const *p = defaults ; p->key ; p++)
   {
-    if (!conftree_search(p->key))
+    size_t n = genalloc_len(node, &conf.list) ;
+    for (size_t i = 0 ; i < n ; i++)
+      if (conf.storage.s[genalloc_s(node, &conf.list)[i].key.left] == 'A') goto cont ;
+  }
+
+  {
+    node *nod = repo_searchs(&conf, "G:listen4") ;
+    if (!nod) nod = repo_searchs(&conf, "G:listen6") ;
+    if (!nod)
     {
-      node node ;
-      confnode_start(&node, p->key, 0, 0) ;
-      confnode_add(&node, p->value, p->vlen) ;
-      conftree_add(&node) ;
+      repo_add_new(&conf, "A4:\b\177\0\0\1", 8, "", 0, 0, 0) ;
+      repo_add_new(&conf, "A6:\200\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1", 20, "", 0, 0, 0) ;
     }
+    else strerr_warnw1x("listen directives without accept directives") ;
   }
+
+ cont:
+  for (struct defaults_s const *p = defaults ; p->key ; p++)
+    if (!repo_searchs(&conf, p->key))
+      repo_adds_new(&conf, p->key, p->value, p->vlen, 0, 0) ;
 }
diff --git a/src/config/deps-exe/shibari-cache-config b/src/config/deps-exe/shibari-cache-config
index 1685f25..581ff6d 100644
--- a/src/config/deps-exe/shibari-cache-config
+++ b/src/config/deps-exe/shibari-cache-config
@@ -1,8 +1,6 @@
-util.o
-node.o
 repo.o
-conftree.o
 defaults.o
 lexparse.o
+util.o
 -ls6dns
 -lskarnet
diff --git a/src/config/lexparse.c b/src/config/lexparse.c
index 9fa3416..250fa62 100644
--- a/src/config/lexparse.c
+++ b/src/config/lexparse.c
@@ -5,6 +5,7 @@
 #include <stdlib.h>
 #include <errno.h>
 
+#include <skalibs/gccattributes.h>
 #include <skalibs/uint16.h>
 #include <skalibs/uint32.h>
 #include <skalibs/fmtscan.h>
@@ -17,20 +18,8 @@
 
 #include <s6-dns/s6dns-domain.h>
 
-#include <shibari/config.h>
 #include "shibari-cache-config-internal.h"
 
-#define dietoobig() strerr_diefu1sys(100, "read configuration")
-
-typedef struct mdt_s mdt, *mdt_ref ;
-struct mdt_s
-{
-  size_t filepos ;
-  uint32_t line ;
-  char linefmt[UINT32_FMT] ;
-} ;
-#define MDT_ZERO { .filepos = 0, .line = 0, .linefmt = "0" }
-
 struct namevalue_s
 {
   char const *name ;
@@ -47,28 +36,50 @@ enum directivevalue_e
   T_FORWARD,
 } ;
 
-static void conftree_checkunique (char const *key, mdt const *md)
+static void dieparse (char const *, uint32_t, char const *, char const *, char const *, uint32_t, uint32_t) gccattr_noreturn ;
+static void dieparse (char const *ifile, uint32_t line, char const *directive, char const *what, char const *key, uint32_t keylen, uint32_t prevline)
 {
-  node const *node = conftree_search(key) ;
-  if (node)
+  unsigned int m = 3 ;
+  stralloc sa = STRALLOC_ZERO ;
+  char const *ar[11] = { "in file ", ifile, " line " } ;
+  char fmtl[UINT32_FMT] ;
+  char fmtp[UINT32_FMT] ;
+  fmtl[uint32_fmt(fmtl, line)] = 0 ;
+  ar[m++] = fmtl ;
+  if (directive)
+  {
+    ar[m++] = " directive " ;
+    ar[m++] = directive ;
+  }
+  ar[m++] = ": " ;
+  ar[m++] = what ;
+  if (key)
   {
-    char fmt[UINT32_FMT] ;
-    fmt[uint32_fmt(fmt, node->line)] = 0 ;
-    if (key[0] == 'A')
-      strerr_diefn(1, 11, "duplicate ", "key in file ", g.storage.s + md->filepos, " line ", md->linefmt, ", previously defined", " in file ", g.storage.s + node->filepos, " line ", fmt) ;
-    else
-      strerr_diefn(1, 12, "duplicate ", "key ", key, " in file ", g.storage.s + md->filepos, " line ", md->linefmt, ", previously defined", " in file ", g.storage.s + node->filepos, " line ", fmt) ;
+    if (!string_quote(&sa, key, keylen) || !stralloc_0(&sa)) dienomem() ;
+    ar[m++] = sa.s ;
   }
+  if (prevline)
+  {
+    fmtp[uint32_fmt(fmtp, prevline)] = 0 ;
+    ar[m++] = " - see line " ;
+    ar[m++] = fmtp ;
+  }
+  strerr_diev(1, ar, m) ;
+}
+
+static void add_unique (char const *ifile, uint32_t line, char const *directive, char const *key, uint32_t keylen, char const *data, size_t datalen)
+{
+  uint32_t prev = repo_add(&conf, key, keylen, data, datalen, line, 0, 0) ;
+  if (prev) dieparse(ifile, line, directive, "duplicate key ", key, keylen, prev) ;
 }
+#define adds_unique(ifile, line, directive, key, data, datalen) add_unique(ifile, line, directive, key, strlen(key), data, datalen)
 
-static void add_unique (char const *key, char const *value, size_t valuelen, mdt const *md)
+static void add_accu (char const *ifile, uint32_t line, char const *directive, char const *key, uint32_t keylen, char const *data, size_t datalen, memcmp_func_ref f)
 {
-  node node ;
-  conftree_checkunique(key, md) ;
-  confnode_start(&node, key, md->filepos, md->line) ;
-  confnode_add(&node, value, valuelen) ;
-  conftree_add(&node) ;
+  uint32_t prev = repo_add(&conf, key, keylen, data, datalen, line, 1, f) ;
+  if (prev) dieparse(ifile, line, directive, "value already listed for key ", key, keylen, prev) ;
 }
+#define adds_accu(ifile, line, directive, key, data, datalen, f) add_accu(ifile, line, directive, key, strlen(key), data, datalen, f)
 
 static int ip40_scan (char const *s, char *ip)
 {
@@ -82,109 +93,104 @@ static int ip60_scan (char const *s, char *ip)
   return len ? !s[len] : 0 ;
 }
 
-static inline void parse_verbosity (char const *s, size_t const *word, size_t n, mdt const *md)
+static int ipcmp (void const *a, void const *b, size_t n)
+{
+  char const *aa = a ;
+  char const * bb = b ;
+  return memcmp(aa+1, bb+1, n-1) ;
+}
+
+
+static inline void parse_verbosity (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line)
 {
   uint32_t v ;
   char pack[4] ;
-  if (n != 1)
-    strerr_dief8x(1, "too ", n ? "many" : "few", " arguments to directive ", "verbosity", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-  if (!uint320_scan(s + word[0], &v))
-    strerr_dief7x(1, " argument to directive ", "verbosity", " must be an integer ", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  if (n != 1) dieparse(ifile, line, "verbosity", n ? "too many arguments" : "too few arguments", 0, 0, 0) ;
+  if (!uint320_scan(s + word[0], &v)) dieparse(ifile, line, "verbosity", "argument must be an integer", 0, 0, 0) ;
   uint32_pack_big(pack, v) ;
-  add_unique("G:logv", pack, 4, md) ;
+  adds_unique(ifile, line, "verbosity", "G:logv", pack, 4) ;
 }
 
-static inline void parse_maxtcp (char const *s, size_t const *word, size_t n, mdt const *md)
+static inline void parse_maxtcp (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line)
 {
   uint32_t max ;
   char pack[4] ;
-  if (n != 1)
-    strerr_dief8x(1, "too ", n ? "many" : "few", " arguments to directive ", "maxtcp", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-  if (!uint320_scan(s + word[0], &max))
-    strerr_dief7x(1, " argument to directive ", "maxtcp", " must be an integer ", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-  if (max > 4000)
-    strerr_dief7x(1, " argument to directive ", "maxtcp", " must be 4000 or less ", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  if (n != 1) dieparse(ifile, line, "maxtcp", n ? "too many arguments" : "too few arguments", 0, 0, 0) ;
+  if (!uint320_scan(s + word[0], &max)) dieparse(ifile, line, "maxtcp", "argument must be an integer", 0, 0, 0) ;
+  if (max > 4000) dieparse(ifile, line, "maxtcp", "argument must be 4000 or less", 0, 0, 0) ;
   uint32_pack_big(pack, max) ;
-  add_unique("G:maxtcp", pack, 4, md) ;
+  adds_unique(ifile, line, "maxtcp", "G:maxtcp", pack, 4) ;
 }
 
-static inline void parse_listen (char const *s, size_t const *word, size_t n, mdt const *md)
+static inline void parse_listen (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line)
 {
-  if (!n)
-    strerr_dief6x(1, "too few arguments to directive ", "listen", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  if (!n) dieparse(ifile, line, "listen", "too few arguments", 0, 0, 0) ;
+  for (size_t i = 0 ; i < n ; i++)
   {
-    size_t n4 = 0, n6 = 0 ;
-    char ip6[n << 4] ;
-    char ip4[n << 2] ;
-    for (size_t i = 0 ; i < n ; i++)
-    {
-      if (ip60_scan(s + word[i], ip6 + (n6 << 4))) n6++ ;
-      else if (ip40_scan(s + word[i], ip4 + (n4 << 2))) n4++ ;
-      else strerr_dief6x(1, "arguments to directive ", "listen", " must be IPs in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-    }
-    add_unique("G:listen4", ip4, n4 << 2, md) ;
-    add_unique("G:listen6", ip6, n6 << 4, md) ;
+    char ip[16] ;
+    char key[10] = "G:listen?" ;
+    if (ip60_scan(s + word[i], ip)) key[8] = '6' ;
+    else if (ip40_scan(s + word[i], ip)) key[8] = '4' ;
+    else dieparse(ifile, line, "listen", "arguments must be IP addresses", 0, 0, 0) ;
+    adds_accu(ifile, line, "listen", key, ip, key[8] == '6' ? 16 : 4, &memcmp) ;
   }
+  adds_accu(ifile, line, "listen", "G:listen4", "", 0, 0) ;
+  adds_accu(ifile, line, "listen", "G:listen6", "", 0, 0) ;
 }
 
-static inline void parse_accept (char const *s, size_t const *word, size_t n, mdt const *md)
+static inline void parse_accept (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line)
 {
-  char key[21] = "A?:" ;
-  if (!n)
-    strerr_dief6x(1, "too few arguments to directive ", "accept", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  char key[20] = "A?:" ;
+  if (!n) dieparse(ifile, line, "accept", "too few arguments", 0, 0, 0) ;
   for (size_t i = 0 ; i < n ; i++)
   {
     uint16_t mask ;
-    uint8_t ipz = 16 ;
-    size_t n = ip6_scan(s + word[i], key + 4) ;
-    if (!n)
+    size_t len = ip6_scan(s + word[i], key + 4) ;
+    if (!len)
     {
-       ipz = 4 ;
-       n = ip4_scan(s + word[i], key + 4) ;
-       if (!n) goto err ;
+       len = ip4_scan(s + word[i], key + 4) ;
+       if (!len) dieparse(ifile, line, "accept", "arguments must be ip/netmask", 0, 0, 0) ;
+       key[1] = '4' ;
     }
-    if (s[word[i] + n] != '/' && s[word[i] + n] != '_') goto err ;
-    if (!uint160_scan(s + word[i] + n + 1, &mask) || mask > (ipz << 3)) goto err ;
-    key[1] = ipz == 16 ? '6' : '4' ;
+    else key[1] = '6' ;
+    if ((s[word[i] + len] != '/' && s[word[i] + len] != '_')
+     || !uint160_scan(s + word[i] + len + 1, &mask)
+     || mask > (key[1] == 6 ? 128 : 32))
+      dieparse(ifile, line, "accept", "arguments must be ip/netmask", 0, 0, 0) ;
     key[3] = (uint8_t)mask ;
-    if (ipz == 16) ip6_netmask(key + 4, mask) ; else ip4_netmask(key + 4, mask) ;
-    add_unique(key, "", 0, md) ;
+    if (key[1] == '6') ip6_netmask(key + 4, mask) ; else ip4_netmask(key + 4, mask) ;
+    add_unique(ifile, line, "accept", key, key[1] == '6' ? 20 : 8, "", 0) ;
   }
- err:
-  strerr_dief6x(1, "arguments to directive ", "accept", " must be IP/mask in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
 }
 
-static inline void parse_server (char const *s, size_t const *word, size_t n, mdt const *md, int forward)
+static inline void parse_server (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line, int forward)
 {
-  char const *x = forward ? "forward" : "server" ;
+  char const *what = forward ? "forward" : "server" ;
   s6dns_domain_t domain ;
-  if (n-- < 2)
-    strerr_dief8x(1, "too ", "few", " arguments to directive ", x, " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-  if (!s6dns_domain_fromstring(&domain, s + word[0], strlen(s + word[0]))
-   || !s6dns_domain_noqualify(&domain))
-    strerr_dief7x(1, "first argument to directive ", x, " must be a zone ", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  char key[258] = "R?:" ;
+  char data[17] ;
+  if (n-- < 2) dieparse(ifile, line, what, "too few arguments", 0, 0, 0) ;
+  if (!s6dns_domain_fromstring_noqualify_encode(&domain, s + word[0], strlen(s + word[0])))
+    dieparse(ifile, line, what, "first argument must be a zone", 0, 0, 0) ;
   word++ ;
+  memcpy(key + 3, domain.s, domain.len - 1) ;
+  for (size_t i = 0 ; i < n ; i++)
   {
-    size_t n4 = 0, n6 = 0 ;
-    char ip6[n * 17] ;
-    char ip4[n * 5] ;
-    char key[3 + domain.len] ;
-    for (size_t i = 0 ; i < n ; i++)
-    {
-      if (ip60_scan(s + word[i], ip6 + (n6 * 17) + 1)) ip6[n6++ * 17] = !!forward ;
-      else if (ip40_scan(s + word[i], ip4 + (n4 * 5) + 1)) ip4[n4++ * 5] = !!forward ;
-      else strerr_dief6x(1, "subsequent arguments to directive ", x, " must be IPs in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
-    }
-    memcpy(key, "R4:", 3) ;
-    memcpy(key + 3, domain.s + 1, domain.len - 1) ;
-    key[2 + domain.len] = 0 ;
-    add_unique(key, ip4, n4 * 5, md) ;
-    key[1] = '6' ;
-    add_unique(key, ip6, n6 * 17, md) ; 
+    if (ip60_scan(s + word[i], data + 1)) key[1] = '6' ;
+    else if (ip40_scan(s + word[i], data + 1)) key[1] = '4' ;
+    else dieparse(ifile, line, what, "second and subsequent arguments must be IP addresses", 0, 0, 0) ;
+    data[0] = !!forward ;
+    add_accu(ifile, line, what, key, 3 + domain.len, data, key[1] == '6' ? 17 : 5, &ipcmp) ;
+  }
+  if (domain.len == 1)
+  {
+    adds_accu(ifile, line, what, "R4:", "", 0, 0) ;
+    adds_accu(ifile, line, what, "R6:", "", 0, 0) ;
   }
 }
 
-static inline void process_line (char const *s, size_t const *word, size_t n, mdt *md)
+
+static inline void process_line (char const *s, size_t const *word, size_t n, char const *ifile, uint32_t line)
 {
   static struct namevalue_s const directives[] =
   {
@@ -200,27 +206,26 @@ static inline void process_line (char const *s, size_t const *word, size_t n, md
   if (!n--) return ;
   word0 = s + *word++ ;
   directive = BSEARCH(struct namevalue_s, word0, directives) ;
-  if (!directive)
-    strerr_dief6x(1, "unrecognized word ", word0, " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  if (!directive) dieparse(ifile, line, 0, "unrecognized word: ", word0, strlen(word0), 0) ;
   switch (directive->value)
   {
     case T_VERBOSITY :
-      parse_verbosity(s, word, n, md) ;
+      parse_verbosity(s, word, n, ifile, line) ;
       break ;
     case T_MAXTCP :
-      parse_maxtcp(s, word, n, md) ;
+      parse_maxtcp(s, word, n, ifile, line) ;
       break ;
     case T_LISTEN :
-      parse_listen(s, word, n, md) ;
+      parse_listen(s, word, n, ifile, line) ;
       break ;
     case T_ACCEPT :
-      parse_accept(s, word, n, md) ;
+      parse_accept(s, word, n, ifile, line) ;
       break ;
     case T_SERVER :
-      parse_server(s, word, n, md, 0) ;
+      parse_server(s, word, n, ifile, line, 0) ;
       break ;
     case T_FORWARD :
-      parse_server(s, word, n, md, 1) ;
+      parse_server(s, word, n, ifile, line, 1) ;
       break ;
   }
 }
@@ -240,13 +245,13 @@ static inline uint8_t cclass (char c)
   }
 }
 
-static inline char next (buffer *b, mdt const *md)
+static inline char next (buffer *b, char const *ifile, uint32_t line)
 {
   char c ;
   ssize_t r = buffer_get(b, &c, 1) ;
   if (r == -1) strerr_diefu1sys(111, "read from preprocessor") ;
   if (!r) return 0 ;
-  if (!c) strerr_dief5x(1, "null character", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ;
+  if (!c) dieparse(ifile, line, 0, "null character", 0, 0, 0) ;
   return c ;
 }
 
@@ -261,12 +266,11 @@ void conf_lexparse (buffer *b, char const *ifile)
   } ;
   stralloc sa = STRALLOC_ZERO ;
   genalloc words = GENALLOC_ZERO ; /* size_t */
-  mdt md = MDT_ZERO ;
+  uint32_t line = 1 ;
   uint8_t state = 0 ;
-  if (!stralloc_catb(&g.storage, ifile, strlen(ifile) + 1)) dienomem() ;
   while (state < 0x04)
   {
-    char c = next(b, &md) ;
+    char c = next(b, ifile, line) ;
     uint8_t what = table[state][cclass(c)] ;
     state = what & 0x07 ;
     if (what & 0x10) if (!genalloc_catb(size_t, &words, &sa.len, 1)) dienomem() ;
@@ -274,11 +278,10 @@ void conf_lexparse (buffer *b, char const *ifile)
     if (what & 0x40) if (!stralloc_0(&sa)) dienomem() ;
     if (what & 0x80)
     {
-      process_line(sa.s, genalloc_s(size_t, &words), genalloc_len(size_t, &words), &md) ;
+      process_line(sa.s, genalloc_s(size_t, &words), genalloc_len(size_t, &words), ifile, line) ;
       genalloc_setlen(size_t, &words, 0) ;
       sa.len = 0 ;
-      md.line++ ;
-      md.linefmt[uint32_fmt(md.linefmt, md.line)] = 0 ;
+      line++ ;
     }
   }
   genalloc_free(size_t, &words) ;
diff --git a/src/config/node.c b/src/config/node.c
deleted file mode 100644
index 7e6cd4b..0000000
--- a/src/config/node.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* ISC license. */
-
-#include <stdint.h>
-#include <string.h>
-
-#include <skalibs/stralloc.h>
-#include <skalibs/strerr.h>
-
-#include "shibari-cache-config-internal.h"
-
-#define diestorage() strerr_diefu2x(100, "add node to configuration tree", ": too much data")
-#define diefilepos() strerr_diefu2x(100, "add node to configuration tree", ": file too large")
-
-void node_start (stralloc *storage, node *node, char const *key, size_t filepos, uint32_t line)
-{
-  size_t l = strlen(key) ;
-  size_t k = storage->len ;
-  if (!stralloc_catb(storage, key, l + 1)) dienomem() ;
-  if (storage->len >= UINT32_MAX) diestorage() ;
-  if (filepos > UINT32_MAX) diefilepos() ;
-  node->key = k ;
-  node->keylen = l ;
-  node->data = storage->len ;
-  node->datalen = 0 ;
-  node->filepos = filepos ;
-  node->line = line ;
-}
-
-void node_add (stralloc *storage, node *node, char const *s, size_t len)
-{
-  if (!stralloc_catb(storage, s, len)) dienomem() ;
-  if (storage->len >= UINT32_MAX) diestorage() ;
-  node->datalen += len ;
-}
diff --git a/src/config/repo.c b/src/config/repo.c
index 8ed8c51..bf847f1 100644
--- a/src/config/repo.c
+++ b/src/config/repo.c
@@ -3,44 +3,120 @@
 #include <stdint.h>
 #include <string.h>
 
+#include <skalibs/diuint32.h>
+#include <skalibs/stralloc.h>
 #include <skalibs/genalloc.h>
 #include <skalibs/avltree.h>
+#include <skalibs/cdbmake.h>
 
 #include "shibari-cache-config-internal.h"
 
-void *node_dtok (uint32_t d, void *data)
+static void *node_dtok (uint32_t d, void *aux)
 {
-  repo *r = data ;
-  return r->storage->s + genalloc_s(node, &r->ga)[d].key ;
+  repo *rp = aux ;
+  return &genalloc_s(node, &rp->list)[d].key ;
 }
 
-int node_cmp (void const *a, void const *b, void *data)
+static int node_cmp (void const *a, void const *b, void *aux)
 {
-  (void)data ;
-  return strcmp((char const *)a, (char const *)b) ;
+  repo *rp = aux ;
+  diuint32 const *ka = a ;
+  diuint32 const *kb = b ;
+  if (ka->right < kb->right) return -1 ;
+  if (ka->right > kb->right) return 1 ;
+  return memcmp(rp->storage.s + ka->left, rp->storage.s + kb->left, ka->right) ;
 }
 
-node const *repo_search (repo const *r, char const *key)
+void repo_init (repo *rp)
 {
+  avltree_init(&rp->tree, 32, 3, 8, &node_dtok, &node_cmp, rp) ;
+}
+
+node *repo_search (repo *rp, char const *key, uint32_t keylen)
+{
+  diuint32 ukey = { .left = rp->storage.len, .right = keylen } ;
   uint32_t i ;
-  return avltree_search(&r->tree, key, &i) ? genalloc_s(node const, &r->ga) + i : 0 ;
+  if (!stralloc_catb(&rp->storage, key, keylen)) dienomem() ;
+  rp->storage.len = ukey.left ;
+  return avltree_search(&rp->tree, &ukey, &i) ? genalloc_s(node, &rp->list) + i : 0 ;
 }
 
-void repo_add (repo *r, node const *nod)
+static int checkunique (memcmp_func_ref f, char const *ar, size_t arlen, char const *ele, size_t elelen)
 {
-  uint32_t i = genalloc_len(node, &r->ga) ;
-  if (!genalloc_append(node, &r->ga, nod)) dienomem() ;
-  if (!avltree_insert(&r->tree, i)) dienomem() ;
+  while (arlen >= elelen)
+  {
+    if (!(*f)(ar, ele, elelen)) return 0 ;
+    ar += elelen ;
+    arlen -= elelen ;
+  }
+  return 1 ;
 }
 
-void repo_update (repo *r, node const *nod)
+void repo_add_new (repo *rp, char const *key, uint32_t keylen, char const *data, size_t datalen, uint32_t line, int accu)
 {
-  uint32_t i ;
-  if (avltree_search(&r->tree, r->storage->s + nod->key, &i))
+  node *nod ;
+  uint32_t n = genalloc_len(node, &rp->list) ;
+  if (!genalloc_readyplus(node, &rp->list, 1)) dienomem() ;
+  nod = genalloc_s(node, &rp->list) + n ;
+  nod->key.left = rp->storage.len ;
+  nod->key.right = keylen ;
+  if (!stralloc_catb(&rp->storage, key, keylen)) dienomem() ;
+  nod->line = line ;
+  nod->data = stralloc_zero ;
+  if (accu)
+  {
+    if (!stralloc_catb(&nod->data, data, datalen)) dienomem() ;
+  }
+  else
+  {
+    nod->data.a = rp->storage.len ;
+    nod->data.len = datalen ;
+    if (rp->storage.len + datalen > UINT32_MAX) diestorage() ;
+    if (!stralloc_catb(&rp->storage, data, datalen)) dienomem() ;
+  }
+  genalloc_setlen(node, &rp->list, n + 1) ;
+  if (!avltree_insert(&rp->tree, n)) dienomem() ;
+}
+
+uint32_t repo_add (repo *rp, char const *key, uint32_t keylen, char const *data, size_t datalen, uint32_t line, int accu, memcmp_func_ref f)
+{
+  node *nod ;
+  if (keylen > UINT32_MAX || datalen > UINT32_MAX) diestorage() ;
+  nod = repo_search(rp, key, keylen) ;
+  if (nod)
   {
-    if (!avltree_delete(&r->tree, r->storage->s + nod->key)) dienomem() ;
-    genalloc_s(node, &r->ga)[i] = *nod ;
-    if (!avltree_insert(&r->tree, i)) dienomem() ;
+    if (!nod->data.s || !accu) return nod->line ;
+    if (f && !checkunique(f, nod->data.s, nod->data.len, data, datalen)) return nod->line ;
+    if (nod->data.len + datalen > UINT32_MAX) diestorage() ;
+    if (!stralloc_catb(&nod->data, data, datalen)) dienomem() ;
   }
-  else repo_add(r, nod) ;
+  else repo_add_new(rp, key, keylen, data, datalen, line, accu) ;
+  return 0 ;
+}
+
+static int node_write (cdbmaker *cm, node *nod, char const *s)
+{
+  return cdbmake_add(cm, s + nod->key.left, nod->key.right, nod->data.s ? nod->data.s : s + nod->data.a, nod->data.len) ;
+}
+
+int repo_write (cdbmaker *cm, repo const *rp)
+{
+  for (size_t i = 0 ; i < genalloc_len(node, &rp->list) ; i++)
+    if (!node_write(cm, genalloc_s(node, &rp->list) + i, rp->storage.s))
+      return 0 ;
+  return 1 ;
+}
+
+#if 0
+static void node_free (node *nod)
+{
+  if (nod->data.s) stralloc_free(&nod->data) ;
+}
+
+void repo_free (repo *rp)
+{
+  avltree_free(&rp->tree) ;
+  genalloc_deepfree(node, &rp->list, &node_free) ;
+  stralloc_free(&rp->storage) ;
 }
+#endif
diff --git a/src/config/shibari-cache-config-internal.h b/src/config/shibari-cache-config-internal.h
index c7b0197..77f3e79 100644
--- a/src/config/shibari-cache-config-internal.h
+++ b/src/config/shibari-cache-config-internal.h
@@ -7,85 +7,70 @@
 #include <string.h>
 #include <stdlib.h>
 
+#include <skalibs/diuint32.h>
 #include <skalibs/buffer.h>
 #include <skalibs/strerr.h>
 #include <skalibs/stralloc.h>
 #include <skalibs/genalloc.h>
-#include <skalibs/cdbmake.h>
 #include <skalibs/avltree.h>
+#include <skalibs/cdbmake.h>
 
 #define dienomem() strerr_diefu1sys(111, "stralloc_catb")
+#define diestorage() strerr_diefu2x(100, "add node to configuration tree", ": too much data")
+
+typedef int memcmp_func (void const *, void const *, size_t) ;
+typedef memcmp_func *memcmp_func_ref ;
 
 typedef struct node_s node, *node_ref ;
 struct node_s
 {
-  uint32_t key ;
-  uint32_t keylen ;
-  uint32_t data ;
-  uint32_t datalen ;
-  uint32_t filepos ;
+  diuint32 key ;
   uint32_t line ;
+  stralloc data ;
 } ;
-#define NODE_ZERO { .key = 0, .keylen = 0, .data = 0, .datalen = 0 }
+#define NODE_ZERO { .key = DIUINT32_ZERO, .line = 0, .data = STRALLOC_ZERO }
 
 typedef struct repo_s repo, *repo_ref ;
 struct repo_s
 {
-  genalloc ga ;
-  avltree tree ;
-  stralloc *storage ;
-} ;
-#define REPO_ZERO { .ga = GENALLOC_ZERO, .tree = AVLTREE_ZERO, .storage = 0 }
-
-struct global_s
-{
   stralloc storage ;
+  genalloc list ; /* node */
+  avltree tree ;
 } ;
-#define GLOBAL_ZERO { .storage = STRALLOC_ZERO }
-
-extern struct global_s g ;
-
-
- /* util */
-
-extern int keycmp (void const *, void const *) ;  /* for any struct starting with a string key */
-#define BSEARCH(type, key, array) bsearch(key, (array), sizeof(array)/sizeof(type), sizeof(type), &keycmp)
-
-
- /* node */
-
-extern void node_start (stralloc *, node *, char const *, size_t, uint32_t) ;
-extern void node_add (stralloc *, node *, char const *, size_t) ;
+#define REPO_ZERO { .storage = STRALLOC_ZERO, .list = GENALLOC_ZERO, .tree = AVLTREE_ZERO }
 
 
  /* repo */
 
-extern void *node_dtok (uint32_t, void *) ;
-extern int node_cmp (void const *, void const *, void *) ;
-extern node const *repo_search (repo const *, char const *) ;
-extern void repo_add (repo *, node const *) ;
-extern void repo_update (repo *, node const *) ;
+extern void repo_init (repo *) ;
+extern node *repo_search (repo *, char const *, uint32_t) ;
+#define repo_searchs(rp, key) repo_search(rp, (key), strlen(key))
+extern void repo_add_new (repo *, char const *, uint32_t, char const *, size_t, uint32_t, int) ;
+#define repo_adds_new(rp, key, data, datalen, line, accu) repo_add_new(rp, key, strlen(key), data, datalen, line, accu)
+extern uint32_t repo_add (repo *, char const *, uint32_t, char const *, size_t, uint32_t, int, memcmp_func_ref) ;
+#define repo_adds(rp, key, data, datalen, line, accu, f) repo_add(rp, key, strlen(key), data, datalen, line, accu, f)
+extern int repo_write (cdbmaker *, repo const *) ;
+extern void repo_free (repo *) ;
 
 
- /* conftree */
+ /* lexparse */
 
-extern void confnode_start (node *, char const *, size_t, uint32_t) ;
-extern void confnode_add (node *, char const *, size_t) ;
+extern void conf_lexparse (buffer *, char const *) ;
 
-extern node const *conftree_search (char const *) ;
-extern void conftree_add (node const *) ;
-extern void conftree_update (node const *) ;
 
-extern int conftree_write (cdbmaker *) ;
+ /* defaults */
 
+extern void conf_defaults (void) ;
 
- /* lexparse */
 
-extern void conf_lexparse (buffer *, char const *) ;
+ /* util */
 
+extern int keycmp (void const *, void const *) ;  /* for any struct starting with a string key */
+#define BSEARCH(type, key, array) bsearch(key, (array), sizeof(array)/sizeof(type), sizeof(type), &keycmp)
 
- /* defaults */
 
-extern void conf_defaults (void) ;
+ /* main */
+
+extern repo conf ;
 
 #endif
diff --git a/src/config/shibari-cache-config.c b/src/config/shibari-cache-config.c
index 0aef2ce..e919c65 100644
--- a/src/config/shibari-cache-config.c
+++ b/src/config/shibari-cache-config.c
@@ -20,7 +20,7 @@
 #define USAGE "shibari-cache-config [ -i textfile ] [ -o cdbfile ] [ -m mode ]"
 #define dieusage() strerr_dieusage(100, USAGE)
 
-struct global_s g = GLOBAL_ZERO ;
+repo conf = REPO_ZERO ;
 
 static inline void conf_output (char const *ofile, unsigned int omode)
 {
@@ -37,7 +37,7 @@ static inline void conf_output (char const *ofile, unsigned int omode)
     unlink_void(otmp) ;
     strerr_diefu2sys(111, "cdmake_start ", otmp) ;
   }
-  if (!conftree_write(&cm))
+  if (!repo_write(&cm, &conf))
   {
     unlink_void(otmp) ;
     strerr_diefu2sys(111, "write config tree into ", otmp) ;
@@ -88,6 +88,8 @@ int main (int argc, char const *const *argv, char const *const *envp)
     argc -= l.ind ; argv += l.ind ;
   }
 
+  repo_init(&conf) ;
+
   {
     int fdr = openc_readb(ifile) ;
     char buf[4096] ;