/*
CSTLEMMA - trainable lemmatiser

Copyright (C) 2002, 2005, 2009  Center for Sprogteknologi, University of Copenhagen

This file is part of CSTLEMMA.

CSTLEMMA is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

CSTLEMMA is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CSTLEMMA; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "comp.h"
#include "graph.h"

/*
ACL 2009 paper:
Icelandic   71.3    1.5 even_better (71,30 1,51 iflg. D:\dokumenter\tvärsök\even_better\icelandic.xls) peen 71,51 1,65 sugar 70,93 1,86 affiksFEW3 71,02 2,16 no pruning
Danish      92.8    0.2 peen sugar: 92,72 0,19 no pruning
Norwegian   87.6    0.3 affiksFEW2 sugar: 86,67 0,68
Greek       90.4    0.4 sugar no pruning
Slovene     86.7    0.3 affiksFEW3 affiksFEW2: 86,23 0,58 sugar: 86,27 0,41 peen:86,13 0,55  0,4
Swedish     92.3    0.3 sugar pruning 1
German      91.46   0.17 sugar no pruning
English     89.0    1.3 sugar pruning 2
Dutch       90.4    0.5 affiksFEW2 sugar: 90,17 0,31    0,3 no pruning
Polish      93.88   0.08 peen sugar: 93,88 0,08 (?) no pruning
*/

#if _NA
// IMPORTANT (20090511) R__NA and W__NA are not updated as sibling rules are 
// added and eat up the training pairs that earlier siblings did not handle.
// This error was detected after having used the weight functions for 
// the ACL-paper.

int comp_fairly_good(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //fairly good, Icelandic 71.270883
//AMBI:
    // French ok 85.767516 ambi1 1.156051 ambi2 0.955414 diff 12.121019 rules 7337.500000 2.731849% cutoff 2
    int A1 = a->W__R           + a->R__R;
    int B1 = b->W__R           + b->R__R;
    int A2 = a->W__R + a->W__W           + a->R__NA;
    int B2 = b->W__R + b->W__W           + b->R__NA;
    int A3 = a->W__R           + a->R__R + a->R__NA;
    int B3 = b->W__R           + b->R__R + b->R__NA;
/*  int A2 = a->R__NA - a->W__NA;
    int B2 = b->R__NA - b->W__NA;
    int A3 = a->W__R - a->R__W;
    int B3 = b->W__R - b->R__W;
*/
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_even_better(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //even better, Icelandic 71.300716
    // BEST Icelandic 71.535870 +/- 1.919590 at 0.9856 of dataset, 17 iterations, 23209.882353 = 40.477646% rules, cutoff = 0
    // Icelandic 71.283167 +/- 1.714260 at 0.9856 of dataset, 17 iterations, 22719.470588 = 39.622376% rules, cutoff = 0, RECURSE == 4
//AMBI:
    // French ok 85.487261 ambi1 1.283439 ambi2 1.050955 diff 12.178344 rules 7360.125000 2.740283% cutoff 2

    int A1 = a->W__R           + a->R__R;// wr + rr
    int B1 = b->W__R           + b->R__R;
    int A2 = a->W__R           + a->R__R + a->R__NA;// wr + rr + rn - r = wr - rw
    int B2 = b->W__R           + b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__W           + a->R__NA;// wr + ww + rn - w = -wn + rn
    int B3 = b->W__R + b->W__W           + b->R__NA;
//    int A2 = a->W__R - a->R__W;
//    int B2 = b->W__R - b->R__W;
//    int A3 = a->R__NA - a->W__NA;
//    int B3 = b->R__NA - b->W__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_affiksFEW3(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    // Icelandic 65.781623, cutoff 1 (old lemmatizer 73.329356, cutoff 0)
    // Icelandic 66.544995 +/- 1.943469 at 0.9856 of dataset, 17 iterations, 11134.176471 = 19.417817% rules, cutoff = 1
    // English 87.863636, cutoff 2 (old 87.954545, cutoff 1)
    // English 87.806061 +/- 1.009323 at 0.9856 of dataset, 15 iterations, 1619.133333 = 2.152101% rules (cutoff = 2)
    // BEST Slovene 86.669776 +/- 0.331106 at 0.9856 of dataset, 9 iterations, 5650.777778 = 2.888237% rules (cutoff = 2)
    // Slovene-ambi (4.23%) 83.165661, cutoff 3 (3550 rules!) (old 82.017103, cutoff 1, 9377 rules) Better than _affiksFEW2, 82.780013, 6656 rules.
    // Danish 90.942165	+/- 0.589437 at 0.9856 of dataset, 5 iterations, 32327.400000 = 5.925881% rules, cutoff = 1
    // German 90.266461 +/-	0.509202 at 0.9856 of dataset, 7 iterations, 21539.428571 = 6.930653% rules, cutoff = 1
    // Greek 89.640779 +/- 0.402079 at 0.9856 of dataset, 5 iterations, 13377.200000 = 2.472132% rules, cutoff = 2
    // Dutch 87.817059 +/- 0.366236 at 0.9856 of dataset, 7 iterations, 23493.571429 = 7.895486% rules, cutoff = 1
    // Norwegian 85.788507 +/- 0.484921 at 0.9856 of dataset, 6 iterations, 14904.000000 = 3.157580% rules, cutoff = 2
    // Polish 93.203365 +/- 0.175436 at 0.9856 of dataset, 2 iterations, 50597.500000 = 1.491153% rules, cutoff = 2
    // Swedish 91.709042 +/- 0.170094 at 0.9856 of dataset, 6 iterations, 4407.666667 = 0.935737% rules, cutoff = 3
//AMBI:
    // French ok 82.754777 ambi1 2.353503 ambi2 1.805732 diff 13.085987 rules 7360.125000 2.740283% cutoff 2

    /* Interesting because it generates far less rules than the above 
       variables, only 20 % more than the old lemmatizer.
       Also interesting is that there are not many leaves with only one
       supporting training pair.
       Yet, the leaves with only one supporter are detrimentous to the overall
       result (cutoff has to be 1 or even 2).
    */
#if 1
    int A1 = a->W__R           + a->R__R + a->R__NA; // Good: previously wrong words got it right. Bad: previously right words got it wrong.
    int B1 = b->W__R           + b->R__R + b->R__NA;
    int A2 = a->W__R           + a->R__R; // Good: any rightly lemmatized word
    int B2 = b->W__R           + b->R__R;
    int A3 = a->W__R + a->W__W           + a->R__NA; // Good: previously right words that didn't match. They may return to the parent.
    int B3 = b->W__R + b->W__W           + b->R__NA; // Bad: previously wrong words that didn't match. They must be handled by siblings.
#else
    int A1 = a->W__R - a->R__W; // Good: previously wrong words got it right. Bad: previously right words got it wrong.
    int B1 = b->W__R - b->R__W;
    int A2 = a->W__R + a->R__R; // Good: any rightly lemmatized word
    int B2 = b->W__R + b->R__R;
    int A3 = a->W__R + a->W__W - a->R__R - a->R__W; // Good: previously right words that didn't match. They may return to the parent.
    int B3 = b->W__R + b->W__W - b->R__R - b->R__W; // Bad: previously wrong words that didn't match. They must be handled by siblings.
#endif
/*
    int A1 = a->W__R - a->R__W; // Good: previously wrong words got it right. Bad: previously right words got it wrong.
    int B1 = b->W__R - b->R__W;
    int A2 = a->W__R + a->R__R; // Good: any rightly lemmatized word
    int B2 = b->W__R + b->R__R;
    int A3 = a->R__NA - a->W__NA; // Good: previously right words that didn't match. They may return to the parent.
    int B3 = b->R__NA - b->W__NA; // Bad: previously wrong words that didn't match. They must be handled by siblings.
*/
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_affiksFEW(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
/**/
    //_affiksFEW
    // Dutch 88.138224, 39943.5 flexrules cutoff 1 (old 89.656164, 47277.75 flexrules, cutoff 1)
    // German 90.266461	+/-	0.509202 at 0.9856 of dataset, 7 iterations, 21539.428571 = 6.930653% rules, cutoff = 1
//AMBI:
    // French ok 82.617834 ambi1 2.455414 ambi2 1.872611 diff 13.054140 rules 7360.125000 2.740283% cutoff 2
    int N = a->W__W + a->W__R + a->W__NA + a->R__W + a->R__R + a->R__NA;

    int A1;
    int B1;
    int A2;
    int B2;
    int A3;
    int B3;

    // good for small numbers:
    if(N < 3)
        {
        A1 = a->W__R           + a->R__R;
        B1 = b->W__R           + b->R__R;
        A2 = a->W__R           + a->R__R + a->R__NA;
        B2 = b->W__R           + b->R__R + b->R__NA;
        A3 = a->W__R + a->W__W           + a->R__NA;
        B3 = b->W__R + b->W__W           + b->R__NA;
/*      A1 = a->W__R + a->R__R;
        B1 = b->W__R + b->R__R;
        A2 = a->W__R - a->R__W;
        B2 = b->W__R - b->R__W;
        A3 = a->R__NA - a->W__NA;
        B3 = b->R__NA - b->W__NA;
*/
        }
    // good for big numbers:
    else
        {
        A1 = a->W__R + a->R__R + a->R__NA; // Good: previously wrong words got it right. Bad: previously right words got it wrong.
        B1 = b->W__R + b->R__R + b->R__NA;
        A2 = a->W__R + a->R__R; // Good: any rightly lemmatized word
        B2 = b->W__R + b->R__R;
        A3 = a->R__NA + a->W__R + a->W__W; // Good: previously right words that didn't match. They may return to the parent.
        B3 = b->R__NA + b->W__R + a->W__W; // Bad: previously wrong words that didn't match. They must be handled by siblings.
/*      A1 = a->W__R - a->R__W; // Good: previously wrong words got it right. Bad: previously right words got it wrong.
        B1 = b->W__R - b->R__W;
        A2 = a->W__R + a->R__R; // Good: any rightly lemmatized word
        B2 = b->W__R + b->R__R;
        A3 = a->R__NA - a->W__NA; // Good: previously right words that didn't match. They may return to the parent.
        B3 = b->R__NA - b->W__NA; // Bad: previously wrong words that didn't match. They must be handled by siblings.
*/
        }
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_affiksFEW2(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //_affiksFEW2
    // (OK) BEST Dutch 90.452096 +/- 0.655431 at 0.9856 of dataset, 7 iterations, 53607.714286 = 18.015948% rules, cutoff = 0
    // (OK) Norwegian 86.776860 +/- 0.642621 at 0.9856 of dataset, 6 iterations, 112374.000000 = 23.807698% rules, cutoff = 0
    // (OK) English 88.424242 +/- 1.191106 at 0.9856 of dataset, 15 iterations, 1383.000000 = 1.838240% rules, cutoff = 2
    // (OK) Icelandic 71.304226 +/- 1.453643 at 0.9856 of dataset, 17 iterations, 25635.000000 = 44.707011% rules, cutoff = 0
    // (OK) German 91.156762 +/- 0.348391 at 0.9856 of dataset, 7 iterations, 48816.571429 = 15.707506% rules, cutoff = 0
    // (OK) Slovene 86.537639 +/- 0.559484 at 0.9856 of dataset, 9 iterations, 40643.444444 = 20.773759% rules, cutoff = 0
    // (OK) Swedish 91.907598 +/- 0.224888 at 0.9856 of dataset, 6 iterations, 27958.000000 = 5.935415% rules, cutoff = 1
    // (OK) Greek 90.741209 +/- 0.312526 at 0.9856 of dataset, 5 iterations, 125306.400000 = 23.156860% rules, cutoff = 0
    // (OK) Danish 92.994605 +/- 0.210674 at 0.9856 of dataset, 5 iterations, 67278.800000 = 12.332763% rules, cutoff = 0
    
    // (?)ALMOST BEST Polish 93.398015 +/- 0.045642 at 0.9856 of dataset, 2 iterations, 165511.500000 = 4.877770% rules, cutoff = 1
//AMBI:
    // French ok 84.194268 ambi1 2.277070 ambi2 1.576433 diff 11.952229 rules 6453.250000 2.402640% cutoff 2

#if 1 // 20090511
    int A1 = a->W__R           + 2*a->R__R + a->R__NA; // good: all words that are lemmatised correctly. bad: all previously right words that got it wrong
    // wr + 2rr + rn - r = ww + rr - rw
    int B1 = b->W__R           + 2*b->R__R + b->R__NA;
    int A2 = a->W__R           +   a->R__R + a->R__NA;
    // wr + rr + rn - r = wr - rw
    int B2 = b->W__R           +   b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__W             + a->R__NA;
    // wr + ww + rn - w = -wn + rn
    int B3 = b->W__R + b->W__W             + b->R__NA;
#else
    int A1 = a->W__R           + a->R__R - a->R__W; // good: all words that are lemmatised correctly. bad: all previously right words that got it wrong
    // wr + 2rr + rn - r = ww + rr - rw
    int B1 = b->W__R           + b->R__R - b->R__W;
    int A2 = a->W__R                     - a->R__W;
    // wr + rr + rn - r = wr - rw
    int B2 = b->W__R                     - b->R__W;
    int A3 = a->W__R + a->W__W - a->R__R - a->R__W;
    // wr + ww + rn - w = -wn + rn
    int B3 = b->W__R + b->W__W - b->R__R - b->R__W;
#endif
/*  int A1 = a->W__R + a->R__R - a->R__W; // good: all words that are lemmatised correctly. bad: all previously right words that got it wrong
    int B1 = b->W__R + b->R__R - b->R__W;
    int A2 = a->W__R - a->R__W;
    int B2 = b->W__R - b->R__W;
    int A3 = a->R__NA - a->W__NA;
    int B3 = b->R__NA - b->W__NA;
*/
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_affiksFEW2org(const vertex * a,const vertex * b)
    {
    // BEST Norwegian 87.494563 +/- 0.217147 at 0.9856 of dataset, 6 iterations, 101814.500000 = 21.570549% rules, cutoff = 0
    // English 88.260606 +/- 0.826699 at 0.9856 of dataset, 15 iterations, 7362.466667 = 9.785960% rules, cutoff = 1
    // Icelandic 70.651411 +/- 1.565857 at 0.9856 of dataset, 17 iterations, 23232.941176 = 40.517860% rules, cutoff = 0
    // German 90.307358 +/- 0.355867 at 0.9856 of dataset, 7 iterations, 50595.857143 = 16.280019% rules, cutoff = 0
    // Dutch 90.274675 +/- 0.462929 at 0.9856 of dataset, 7 iterations, 23452.142857 = 7.881563% rules, cutoff = 1    
    // Slovene 86.417162 +/- 0.540735 at 0.9856 of dataset, 9 iterations, 40847.666667 = 20.878142% rules, cutoff = 0
    // Swedish 91.982663 +/- 0.250703 at 0.9856 of dataset, 6 iterations, 28998.000000 = 6.156204% rules, cutoff = 1
    // Greek 90.258032 +/- 0.234665 at 0.9856 of dataset, 5 iterations, 43156.000000 = 7.975310% rules, cutoff = 1 (but exactly the same as cutoff = 0)
    // Danish 92.425041 +/- 0.374415 at 0.9856 of dataset, 5 iterations, 73177.800000 = 13.414099% rules, cutoff = 0
//AMBI:
    // French ok 84.761146 ambi1 2.015924 ambi2 1.665605 diff 11.557325 rules 7262.500000 2.703935% cutoff 2
    int A1 = a->W__R + a->R__R - a->R__W; // good: all words that are lemmatised correctly. bad: all previously right words that got it wrong
    int B1 = b->W__R + b->R__R - b->R__W;
    int A2 = a->W__R - a->R__W;
    int B2 = b->W__R - b->R__W;
    int A3 = a->R__NA - a->W__NA;
    int B3 = b->R__NA - b->W__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_fixNA(const vertex * a,const vertex * b)
    {
    /*
    Icelandic 47.982267 (at 0.8488 of dataset)
    */
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //_fixNA
//AMBI:
    // French: stopped because of very bad results. (> 25% wrong results) 

    int A1 = a->W__R + a->W__NA           + a->R__NA;
    int B1 = b->W__R + b->W__NA           + b->R__NA;
    int A2 = a->W__R            + a->R__R + a->R__NA;
    int B2 = b->W__R            + b->R__R + b->R__NA;
    int A3 = a->W__R                      + a->R__NA;
    int B3 = b->W__R                      + b->R__NA;

    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_fruit(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 71.344041 at 0.939 of dataset
    //ALMOST BEST Icelandic 71.521831 +/- 1.988737 at 0.9856 of dataset, 17 iterations, 23539.352941 = 41.052237% rules
    //Slovene 85.900276 +/- 0.456532 at 0.9856 of dataset, 9 iterations, 42167.333333 = 21.552652% rules
    //English 87.626771	+/- 0.060148 at 0.4928 (!) of dataset, 3 iterations, 933.000000 = 2.480262% rules
//AMBI:
    // French ok 85.382166 ambi1 1.359873 ambi2 1.089172 diff 12.168790 rules 7259.125000 2.899075% cutoff 2
    int A1 = a->W__R            + a->R__R;
    int B1 = b->W__R            + b->R__R;
    int A2 = a->W__R            + a->R__R + a->R__NA;
    int B2 = b->W__R            + b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_ice(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 60.242322 at 0.939 of dataset
//AMBI:
    // French ok 82.557325 ambi1 2.522293 ambi2 1.866242 diff 13.054140 rules 8556.625000 3.185757% cutoff 2
    int A1 = a->W__R            + a->R__R + a->R__NA;
    int B1 = b->W__R            + b->R__R + b->R__NA;
    int A2 = a->W__R            + a->R__R;
    int B2 = b->W__R            + b->R__R;
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_pisang(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 71.287687 at 0.939 of dataset
//AMBI:
    // French ok 85.414013 ambi1 1.359873 ambi2 1.085987 diff 12.140127 rules 7848.375000 2.922065% cutoff 2
    int A1 = a->W__R            + a->R__R;
    int B1 = b->W__R            + b->R__R;
    int A2 = a->W__R + a->W__NA           + a->R__NA;
    int B2 = b->W__R + b->W__NA           + b->R__NA;
    int A3 = a->W__R            + a->R__R + a->R__NA;
    int B3 = b->W__R            + b->R__R + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_kiwi(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 70.865032 at 0.939 of dataset
//AMBI:
    // French ok 85.410828 ambi1 1.378981 ambi2 1.035032 diff 12.175159 rules 7676.875000 2.858213% cutoff 2
    int A1 = a->W__R            + a->R__R;
    int B1 = b->W__R            + b->R__R;
    int A2 = a->W__R + a->W__NA + a->R__R + a->R__NA;
    int B2 = b->W__R + b->W__NA + b->R__R + b->R__NA;
    int A3 = a->W__R                      + a->R__NA;
    int B3 = b->W__R                      + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_carrot(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 71.090448 at 0.939 of dataset
//AMBI:
    // French ok 85.060510 ambi1 1.328025 ambi2 1.041401 diff 12.570064 rules 7241.625000 2.696163% cutoff 2
    int A1 = 4*(a->W__R        + a->R__R) + a->R__NA;
    int B1 = 4*(b->W__R        + b->R__R) + b->R__NA;
    int A2 = a->W__R            + a->R__R + a->R__NA;
    int B2 = b->W__R            + b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_peen(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    // Icelandic 71.344041 at 0.939 of dataset
    // ALMOST BEST Icelandic 71.507792 +/- 1.645702 at 0.9856 of dataset, 17 iterations, 25240.882353 = 44.019676% rules
    // Slovene 86.133458	+/- 0.549185 at 0.9856 of dataset, 9 iterations, 40898.777778 = 20.904266% rules
    // English 87.803261	+/- 0.106156 at 0.4928 (!) of dataset, 3 iterations, 889.333333 = 2.364179% rules
    // Dutch 89.837692 +/- 0.412795 at 0.9856 of dataset, 7 iterations, 56640.285714 = 19.035104% rules, cutoff = 0
    // ALMOST BEST German 91.288892 +/- 0.670828 at 0.9856 of dataset, 7 iterations, 50584.857143 = 16.276480% rules, cutoff = 0
    // Swedish 91.873698 +/- 0.367967 at 0.9856 of dataset, 6 iterations, 9066.166667 = 1.924725% rules, cutoff = 2
    // ALMOST BEST Norwegian 87.535644 +/- 0.344659 at 0.9856 of dataset, 6 iterations, 48468 = 10.268492% rules, cutoff = 1
    // ALMOST BEST Greek 90.414875+/- 0.385254 at 0.9856 of dataset, 5 iterations, 120691.4 = 22.303999% rules, cutoff = 0
    // BEST Danish 92.796387 +/- 0.214267 at 0.9856 of dataset, 5 iterations, 67807 = 12.429587% rules, cutoff = 0
    // ALMOST BEST Russian 80.484806 +/- 0.409391 at 0.9856 of dataset, 6 iterations, 54630 = 14.022614% rules, cutoff = 1
    // BEST Polish 93.880103 +/- 0.077021 at 0.9856 of dataset, 2 iterations, 344944.5	= 10.165818% rules, cutoff = 0
//AMBI:
    // French ok 84.993631 ambi1 1.388535 ambi2 1.085987 diff 12.531847 rules 7318.375000 2.724738% cutoff 2
    int A1 = 3*(a->W__R        + a->R__R) + a->R__NA;
    // 3wr + 3rr + rn - r = 3wr + 2rr - rw
    int B1 = 3*(b->W__R        + b->R__R) + b->R__NA;
    int A2 = a->W__R            + a->R__R + a->R__NA;
    //wr + rr + rn - r = wr - rw
    int B2 = b->W__R            + b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    // wr + wn + rn - w = -ww + rn
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif
#if _NA
int comp_sugar(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    // Slovene 86.273367 +/- 0.410931 at 0.9856 of dataset, 9 iterations, 17254.777778 = 8.819297% rules (cutoff = 1)
    // BEST English 89.060606 +/- 1.320829 at 0.9856 of dataset, 3 iterations, 1318.266667 = 1.752199% rules, cutoff=2
    // Icelandic 70.925172 +/- 1.858255 at 0.9856 of dataset, 17 iterations, 27151.294118 = 47.351402% rules, cutoff = 0
    // Dutch 90.172822 +/- 0.307911 at 0.9856 of dataset, 7 iterations, 57761.142857 = 19.411791% rules, cutoff = 0
    // BEST Greek 90.422464 +/- 0.437009 at 0.9856 of dataset, 5 iterations, 132765.6 = 24.535334% rules, cutoff = 0
    // BEST German 91.461918 +/- 0.167574 at 0.9856 of dataset, 7 iterations, 50986 = 16.405554% rules, cutoff = 0
    // BEST Swedish 92.265969 +/- 0.277289 at 0.9856 of dataset, 6 iterations, 25935.333333 = 5.506008% rules, cutoff = 1
    // Norwegian 86.665700 +/- 0.676264 at 0.9856 of dataset, 6 iterations, 46685.5 = 9.890849% rules, cutoff = 1
    // Danish 92.585623 +/- 0.171327 at 0.9856 of dataset, 5 iterations, 30422.400000 = 5.576679% rules, cutoff = 1
    // BEST Russian 80.815622 +/- 0.450500 at 0.9856 of dataset, 6 iterations, 47079.166667 = 12.084440% rules, cutoff = 1		
//AMBI:
    // French ok 75.472316 ambi1 4.615600 ambi2 3.493266 diff 16.418818 rules 4162.909091 3.129560% cutoff 2
#if 1
    // next lines from affixFEW2
    int A1 = a->W__R           + 2*a->R__R + a->R__NA; // good: all words that are lemmatised correctly. bad: all previously right words that got it wrong
    // wr - rw + rr
    int B1 = b->W__R           + 2*b->R__R + b->R__NA;
    int A2 = a->W__R           +   a->R__R + a->R__NA;
    // wr - rw
    int B2 = b->W__R           +   b->R__R + b->R__NA;
    // next lines from peen
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    // -ww + rn 
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
#else
    //equivalent with:
    int AA1 = a->W__R - a->R__W + a->R__R;
    int AA2 = - a->R__R;
    int AA3 = - a->W__R - 2*a->W__W;//a->R__NA - a->W__W;
    int BB1 = b->W__R - b->R__W + b->R__R;
    int BB2 = - b->R__R;
    int BB3 = - b->W__R - 2*b->W__W;//b->R__NA - b->W__W;
#endif
    }
#endif

int comp_honey(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    // (OK) Dutch 90.179393 +/- 0.589662 at 0.9856 of dataset, 7 iterations, 73324.571429 = 24.642193% rules, cutoff = 0
    // (OK) Norwegian 87.272244 +/- 0.267729 at 0.9856 of dataset, 6 iterations, 141038.666667 = 29.880630% rules, cutoff = 0
    // (OK) English 88.315152 +/- 1.097312 at 0.9856 of dataset, 3 iterations, 5285.466667 = 7.025276% rules, cutoff=1
    // (OK) Icelandic 70.742665 +/- 1.686147 at 0.9856 of dataset, 17 iterations, 29857.000000 = 52.070108% rules, cutoff = 0
    // (?) Slovene 86.273367 +/- 0.410931 at 0.9856 of dataset, 9 iterations, 17254.777778 = 8.819297% rules (cutoff = 1)
    // (?) BEST Greek 90.422464 +/- 0.437009 at 0.9856 of dataset, 5 iterations, 132765.6 = 24.535334% rules, cutoff = 0
    // (?) BEST German 91.461918 +/- 0.167574 at 0.9856 of dataset, 7 iterations, 50986 = 16.405554% rules, cutoff = 0
    // (?) BEST Swedish 92.265969 +/- 0.277289 at 0.9856 of dataset, 6 iterations, 25935.333333 = 5.506008% rules, cutoff = 1
    // (?) Danish 92.585623 +/- 0.171327 at 0.9856 of dataset, 5 iterations, 30422.400000 = 5.576679% rules, cutoff = 1
    // (?) BEST Russian 80.815622 +/- 0.450500 at 0.9856 of dataset, 6 iterations, 47079.166667 = 12.084440% rules, cutoff = 1		
//AMBI:
    // French ok 84.477707 ambi1 2.251592 ambi2 1.426752 diff 11.843949 rules 7413.875000 2.760295% cutoff 2
    int A1 = a->W__R           + 2*a->R__R;
    int B1 = b->W__R           + 2*b->R__R;
    int A2 = a->W__R           +   a->R__R;
    int B2 = b->W__R           +   b->R__R;
    int A3 = a->W__R ;
    int B3 = b->W__R ;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }

#if _NA
int comp_beet(const vertex * a,const vertex * b)
    {
    //const vertex * a = *(const vertex **)A;
    //const vertex * b = *(const vertex **)B;
    //Icelandic 71.034094 at 0.939 of dataset
//AMBI:
    // French ok 85.057325 ambi1 1.283439 ambi2 1.057325 diff 12.601911 rules 7260.375000 2.703144% cutoff 2
    int A1 = 2*(a->W__R        + a->R__R) + a->R__NA;
    int B1 = 2*(b->W__R        + b->R__R) + b->R__NA;
    int A2 = a->W__R            + a->R__R + a->R__NA;
    int B2 = b->W__R            + b->R__R + b->R__NA;
    int A3 = a->W__R + a->W__NA           + a->R__NA;
    int B3 = b->W__R + b->W__NA           + b->R__NA;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }
#endif

int comp_koud(const vertex * a,const vertex * b)
    {
    // German 91.260578 +/- 0.363285 at 0.9856 of dataset, 7 iterations, 30890.714286 = 9.939577% rules, cutoff = 0
//AMBI:
    // French 86.356688 ambi1 0.996815 ambi2 0.796178 diff 11.850318 rules 3335.250000 1.241763% cutoff 3 (!)
    // French 85.250493 ambi1 2.333057 ambi2 2.161181 diff 10.255268 rules 28520.250000 10.618597% cutoff 0 (!) paradigms+homographs clumped
    // French 85.313973 ambi1 2.050694 ambi2 2.289517 diff 10.345816 rules 28509.250000 10.614432% cutoff 0 (!) homographs clumped
    // Dutch.clumped.ph 85.789838 ambi1 1.086067 ambi2 1.256060 diff 11.868035 rules 37400.142857 12.190637% cutoff 0 paradigms+homographs clumped
    // Dutch.clumped.h 85.818923 ambi1 1.095507 ambi2 1.060476 diff 12.025095 rules 37411.857143 12.192383% cutoff 0 homographs clumped
    // (Dutch.clumped.ph suffix, old algo:
    //                  83.532708 ambi1 1.948624 ambi2 2.719889 ambi3 0.107033 diff 11.691746 rules 73024.571429 23.802477% cutoff 0 paradigms+homographs clumped
    // (Dutch.clumped.h suffix, old algo:
    //                  83.624725 ambi1 1.859813 ambi2 2.611382 ambi3 0.162415 diff 11.741664 rules 72975.428571 23.782417% cutoff 0 paradigms+homographs clumped
    // Russian clumped ph 74.983460 ambi1 0.517762 ambi2 0.558033 diff 23.940745 rules 95077.500000 24.184389% cutoff 0 paradigms+homographs clumped
    // (old algo:)
    //                  79.485114 ambi1 0.218611 ambi2 0.342298 ambi3 0.005753 diff 19.948224 rules 94247.166667 23.973181% cutoff 0 paradigms+homographs clumped
    // The A1 vs B1 condition is pretty close to what later was found as the
    // best factors using automatic factor setting (comp_parms).
    // These factors were found by manual optimizing.
    int A1 = 6*a->W__R - 5*a->R__W + a->W__W;
    int B1 = 6*b->W__R - 5*b->R__W + b->W__W;
    int A2 = a->W__R - 6*a->R__W;
    int B2 = b->W__R - 6*b->R__W;
    int A3 = a->R__R - a->W__W;
    int B3 = b->R__R - b->W__W;
    return (A1>B1)?-1:(A1<B1)?1:(A2>B2)?-1:(A2<B2)?1:(A3>B3)?-1:(A3<B3)?1:0;
    }

// You can find a local optimum for the parameters by using comp_parms as the
// weight function and setting compute_parms = true. The parameters parms[] 
// can be seeded with non-zero values by hard coding. The file parms.txt
// will contain the currently best parameters.

// Optimal parameters == resulting in smallest rule set.
// Hypothesis: small rule sets give (almost) best lemmatization results.
// Optimizing for the size of rule sets is computationally MUCH cheaper
// than optimizing for accuracy.

// If you have found a good set of parameters (presumably with a small subset
// of the training data), you can hard code them (as is done below) and run 
// the program with the full set of the training data. In that case,
// set compute_parms = false

bool compute_parms = false;


#define NPARMS 16
#define ROWPARMS 4
int parms[NPARMS]    = {0};

struct bestParms
    {
    bool useoffset;
    bool suffixonly;
    char * langbase;
    int rows;
    int val[NPARMS];
    };

bestParms  best_sl = 
    {
/*
*/
    true,
    false,
    "sl",
    3,
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     no
cutoff          0
fraction        9856.000000
iterations      9
trainlines      204030.000000
rules           29635.333333
rules%         14.524988
same%stdev     0.941088
ambi1%stdev    0.196612
ambi2%stdev    0.591892
ambi3%stdev    0.000000
diff%stdev     1.043102
same%          80.332522
ambi1%         3.612770
ambi2%         3.022930
ambi3%         0.000000
diff%          13.031778
amb.rules%     7.605250
false_amb%     3.962988
false_not_amb% 5.076311
true_amb%      3.642262
true_not_amb%  87.318440
precision       0.314850
recall          0.417759
*/
//OnlyZeros 12 
//suffix only no 
//iteration:19.58
/* 21083 */
        {
            0,  505, -663,   20, //905780
          -77,  856,-1263,  297, //161407
         -768,  640,-1024,  768, //1542
          512, -512,-1536, 1024  //0
                                 //        0
        }
    };

bestParms best_sl_suffix =
    {
/*
*/
    true,
    true,
    "sl",
    3,
//OnlyZeros 12 
//suffix only yes 
//iteration:19.55
/* 26531 */
        {
          763,  537,  -90, -258, //444000
          540, 1508,-1252,   38, //38754
        -1488, -704,-1792,  272, //5586
        -3072,    0,    0,-1024  //0
                                 //        0
        }
    };


bestParms  best_fr = 
    {
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     no
cutoff          1
fraction        9856.000000
iterations      8
trainlines      268628.750000
rules           16521.750000
rules%         6.150403
same%stdev     0.405192
ambi1%stdev    0.214864
ambi2%stdev    0.273387
ambi3%stdev    0.000000
diff%stdev     0.627035
same%          85.834674
ambi1%         2.640720
ambi2%         1.949180
ambi3%         0.000000
diff%          9.575426
amb.rules%     5.062721
false_amb%     3.351560
false_not_amb% 6.510132
true_amb%      1.711161
true_not_amb%  88.427147
precision       0.203364
recall          0.208138
*/
    true,
    false,
    "fr",
    3,
//OnlyZeros 12 
//suffix only no 
//iteration:19.63
/* 19238 */
        {
            0,  262, -304,   47, //728984
            3,  433, -833,   19, //139658
           16,  288,-1440,  144, //550
        -1664, -256,-2304,  384  //0
                                 //        0
        }
    };

bestParms best_fr_suffix =
    {
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     yes
cutoff          0
fraction        9856.000000
iterations      8
trainlines      268628.750000
rules           39414.250000
rules%         14.672387
same%stdev     0.355492
ambi1%stdev    0.324656
ambi2%stdev    0.255550
ambi3%stdev    0.000000
diff%stdev     0.644838
same%          87.204889
ambi1%         2.132518
ambi2%         1.247990
ambi3%         0.000000
diff%          9.414603
amb.rules%     3.618527
false_amb%     2.196848
false_not_amb% 6.799614
true_amb%      1.421679
true_not_amb%  89.581859
precision       0.244469
recall          0.172926
*/
    true,
    true,
    "fr",
    3,
//OnlyZeros 12 
//suffix only yes 
//iteration:19.60
/* 22512 */
        {
            0,  470, -537,   28, //308198
          219,  255,  -43,    2, //74208
           -4,   89, -112,   27, //6879
         -224,  576,-1664,  480  //0
                                 //        0
        }
    };


bestParms  best_sv = 
    {
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     no
cutoff          2
fraction        9856.000000
iterations      6
trainlines      471038.000000
rules           6864.166667
rules%         1.457243
same%stdev     0.374487
ambi1%stdev    0.000000
ambi2%stdev    0.000000
ambi3%stdev    0.000000
diff%stdev     0.374487
same%          92.256284
ambi1%         0.000000
ambi2%         0.000000
ambi3%         0.000000
diff%          7.743716
amb.rules%     0.000000
false_amb%     0.000000
false_not_amb% 0.000000
true_amb%      0.000000
true_not_amb%  100.000000
precision       0.000000
recall          nan
*/
    true,
    false,
    "sv",
    3,
        {
        //OnlyZeros 12 
        //iteration:19.62
        /* 33474 */
           0, 691, -793, 120,//971415
           2, 586,-1540, 440,//143569
        -432, 512,-1024,  16 //1224
        -448,1024, -512,1024 //0
                             //0
        }
    };

bestParms best_sv_suffix =
    {
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     yes
cutoff          0
fraction        9856.000000
iterations      6
trainlines      471038.000000
rules           51760.833333
rules%         10.988675
same%stdev     0.400911
ambi1%stdev    0.000000
ambi2%stdev    0.000000
ambi3%stdev    0.000000
diff%stdev     0.400911
same%          91.917284
ambi1%         0.000000
ambi2%         0.000000
ambi3%         0.000000
diff%          8.082716
amb.rules%     0.000000
false_amb%     0.000000
false_not_amb% 0.000000
true_amb%      0.000000
true_not_amb%  100.000000
precision       0.000000
recall          nan
*/
    true,
    true,
    "sv",
    3,
        {
        //OnlyZeros 12 
        //iteration:19.58
        /* 41253 */
           0, 608, -608,  1,//285302
          -1, 600, -584, 80,//91856
         -82, 450,-1022,334 //17269
        -448,-512,-2560,512 //0
                            //0
        }
    };

bestParms best_is_suffix =
    {
    true,
    true,
    "is",
    3,
        {
// BEST Icelandic 75.010529 +/- 1.424540 at 0.9856 of dataset, 17 iterations, 19041.705882 = 33.208416% rules, cutoff = 0
// (old)Icelandic 73.620665 +/- 1.337148 at 0.9856 of dataset, 17 iterations, 18363.235294 = 32.025175% rules, cutoff = 0
/*
     -7,  41, -24,  26,
    -80,   0, -56, -24
*/

//iteration:331
/* 18952 */
// 74.877159 +/- 1.394266 at 0.9856 of dataset, 17 iterations, 18736.588235 = 32.676296% rules, cutoff = 0
    /*
0,       0,       0,       0,//            0
0,       0,       0,       0,//            0
0,       6,       -4,      2,//       229741
-10,     -6,      -11,     1 //        18595
*/
//OnlyZeros 12 
//iteration:19.58
/* 16365 */
    -15,269, -174,126,//143775
    416,589, -136, 51,//41148
    -30,528,-2044,215,//5557
    //0,768,-2304,512 //0
    /*
cutoff          0
fraction        9856.000000
iterations      17
trainlines      57340.000000
rules           19312.470588
rules%         33.680625
same%stdev     1.273580
ambi1%stdev    0.000000
ambi2%stdev    0.000000
ambi3%stdev    0.000000
diff%stdev     1.273580
same%          75.396603  (ACL, old algo: 73.2±1.4)
ambi1%         0.000000
ambi2%         0.000000
ambi3%         0.000000
diff%          24.603397
amb.rules%     0.000000
false_amb%     0.000000
false_not_amb% 0.000000
true_amb%      0.000000
true_not_amb%  100.000000
precision       0.000000
recall          nan
*/				    //0
        }
    };

bestParms best_is =
    {
    true,
    false,
    "is",
    3,
        {
// Icelandic 74.631476 +/- 1.476088 at 0.9856 of dataset, 17 iterations, 20398.823529 = 35.575207% rules, cutoff = 0 SUFFIXONLY == 1 during production, SUFFIXONLY == 0 during parameter setting
/*
     0,  34, -30,   7,
    55, -11,  19,  65
*/
//iteration:283
/* 17487 */
    /*
0,       0,       0,       0,//       0
0,       0,       0,       0,//       0
0,       9,       -7,      2,//       636373
7,       4,       -12,     9,//       42035
*/
//OnlyZeros 12 
//iteration:16.35
/* 8192 */
    0  ,  0,   0,  0,//0
    0  ,267,-266, 63,//287175
    7  ,261,-125,449,//14213
    //569,955,-579, 63 //0
        }
    };

bestParms best_ru =
    {
    true,
    false,
    "ru",
    3,
        {
    /*
     0, 10, -10,  1,
     0,  0,   0,  8,
    10,  0,  12,  0
    */
    // After introduction of parmsoff:
    //iteration:1021
    /* 33328 */
    /*
      0,256,-280, 26,//1185047
      1,277,-412, 24,//116109
    -32,176,-272,128,//1709
    */

//iteration:1215
/* 62652 */
       0,1023,-1117,107,//2139379
       9,1115,-1645, 95,//229605
    -121, 709,-1085,513,//3330
//263,1285,-1021,1537//0
        }
    };

bestParms best_ru_suffix =
    {
    true,
    true,
    "ru",
    3,
        {
    // After introduction of parmsoff:
    //iteration:1085
    /* 45283 */
/*
     0,  53, -59,   2,//530032
     0, 146,-181,   0,//150055
     0, 363,-512, 116,//35222
    71,-379,-1021,449 //38711
*/
//iteration:1201
/* 69498 */
/*
cutoff          0
fraction        0.985600
iterations      6
trainlines      393185.000000 
rules           88034.333333 
rules%          22.390054
same%stdev      0.400184 
ambi1%stdev     0.084321 
ambi2%stdev     0.020349
ambi3%stdev     0.000000
diff%stdev      0.347374
same%           81.256165
ambi1%          0.272701
ambi2%          0.223383
ambi3%          0.000000 
diff%           18.247752
amb.rules%      0.696258 
false_amb%      0.646939 
false_not_amb%  1.784160 
true_amb%       0.049318 
true_not_amb%   97.519582
precision       0.036717
recall          0.026899
*/
    0,103, -116,  4,//787020
    0,290, -364,  1,//244334
    5,721,-1028,230,//91786
//135,-763,-2045,897//0
        }
    };

bestParms best_nl =
    {
    true,
    false,
    "nl",
    3,
        {
  /*
    0,   9,  -8,   2,
    7,  11,   9,   9
  */
// After introduction of parmsoff:
//iteration:1267
/* 12787 */
    // 90.294388 +/- 0.606321 at 0.9856 of dataset, 7 iterations, 18187.000000 = 6.112106% rules, cutoff = 1
     0,     738,     -926,   83,    //  463375
    -1,      16,     -255,    6,    //   65235
     8,     544,    -1144,   80     //     726
        }
    };


bestParms best_nl_suffix =
    {
    true,
    true,
    "nl",
    3,
        {
// After introduction of parmsoff:
//iteration:578
/* 16759 */
    0,       5,      -5,    1,   //289215
    0,      31,     -31,    1,   // 21341
    0,      24,     -47,    1,   // 10435
        }
    };

bestParms best_da_suffix =
    {
    false,
    true,
    "da",
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     yes
cutoff          0
fraction        9856.000000
iterations      5
trainlines      585173.000000
rules           61171.600000
rules%         10.453592
same%stdev     0.420111
ambi1%stdev    0.074450
ambi2%stdev    0.108916
ambi3%stdev    0.000000
diff%stdev     0.413541
same%          92.129861
ambi1%         0.480438
ambi2%         0.399572
ambi3%         0.000000
diff%          6.990130
amb.rules%     0.963254
false_amb%     0.582709
false_not_amb% 11.532881
true_amb%      0.380545
true_not_amb%  87.503865
precision       0.246154
recall          0.031943
/var/csttools/bin/makeaffixrules -i STOposUTF8.ph -c 0 -o flexrules.suffix.notags -e da -n 123 -f parms0 -s
*/
//OnlyZeros 12 
//suffix only yes 
//iteration:11.33
/* 11268 */
    3,
        {
            7,   31,  -24,    1, //94608
            1,   45,  -83,    4, //20473
           -3,   26,  -36,   14, //4091
          //-24,   48, -112,   32  //0
                                 //        0
        }

    };

bestParms best_da =
    {
    false,
    false,
    "da",
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     no
cutoff          1
fraction        9856.000000
iterations      5
trainlines      585173.000000
rules           33936.600000
rules%         5.799413
same%stdev     0.375131
ambi1%stdev    0.238159
ambi2%stdev    0.961303
ambi3%stdev    0.000000
diff%stdev     1.218753
same%          88.336306
ambi1%         1.198716
ambi2%         1.648234
ambi3%         0.000000
diff%          8.816744
amb.rules%     3.082412
false_amb%     2.321322
false_not_amb% 11.152337
true_amb%      0.761089
true_not_amb%  85.765252
precision       0.140845
recall          0.063885
*/
//OnlyZeros 12 
//suffix only no 
//iteration:11.61
/* 10481 */
     3,
        {
            0,   14,  -16,    2, //407715
           -2,   19,  -42,   12, //66486
            1,   45,  -50,   19, //317
//            8,    0,  -96,   88  //0
                                 //        0
        }

    };

/* Based on 10000 words
int best_en[]       =
    {
    0, 9,-5, 4,
    0,-4,-5, 0,
    1, 0, 0,-1
    };
*/
bestParms best_en =
    {
    true,
    false,
    "en",
    3,
        {
/* 9986 */
    /* Slightly worse than the values below!
    0,  8, -9,  2,
    1,-10, 11,  0,
    0,  4,-13,  0
    */
/* 10251 */
// 88.763636 +- 1.328464, cutoff 2, 1477.666667 rules, 15 iterations
/*
    0, 4,-5, 1,
    0,-5, 6, 0,
    0, 1, 0, 0
*/
/* 10397 20091021, using 100% of available training data, after 211 iterations */
// 88.484848 +- 1.471358, cutoff 2, 1672.200000 rules, 15 iterations
/*    
    0, 4,-3, 1,
    5, 4,-9,-4,*/
/* 10381 */
// iteration:395
/*
    0,31,-25,8,     //443260
    16,14,-36,-10,  //52623
*/
/* 9859 */
/*  0,35,-36, 2,
    1, 0,  5, 2,
    0,31,-40, 0
*/
// After introduction of parmsoff:
//iteration:94
/* 10398 */
/* 10398 */
/*
    0, 4,-3, 1,// 437782
    2, 1,-4,-2,// 58708
    7, 5, 3, 1,// 4
*/
//iteration:811
/* 10374 */
// 88.466667 +- 1.510243, cutoff 2, 1671.600000 = 2.221838% rules, 15 iterations
    0,  0,   0, 0,  //0
    0,123,-101,30,  //355497
    7, 77, -63,47   //38334
        }
    };

bestParms best_en_suffix =
    {
    true,
    true,
    "en",
    4,
        {
    /* 14144, after 88 iterations*/
// 87.993939 +- 1.039632, cutoff 3, 1156.666667 rules, 15 iterations 
    /*
     1, 3, 0, 1,//421555
    -1, 2,-2, 1,//11288
     1, 6, 3, 0,//19
     */
// After introduction of parmsoff:
//iteration:258
/* 11917 */
/*
    0,  0,  0,  0,// 0
    0,  0,  0,  0,// 0
    0,  4,  0,  4,// 349160
    0,  7,-10, -2 // 22726
*/
//iteration:350
/* 11888 */
// 88.315152 +- 1.084324, cutoff 2, 1650.466667 = 2.193748% rules, 15 iterations    
     0, 1,  0, 1, // 105490
     0, 1,  0, 1, //  13535
     0, 9, -7, 3, //  24911
    -7,-8,-12, 4  //   9828
        }
    };

bestParms best_ge =
    {
    true,
    false,
    "ge",
    3,
        {
//based on 77566 pairs (deutsch_affix_2464_3.UTF8.txt)
//iteration:83
/* 11434 */
/*
    0, 5,-5, 0,//425724
    0, 3,-1, 2,//142447
    7, 5, 2, 1//51604
*/
// After introduction of parmsoff:
//iteration:322
/* 10970 */       //         
/*
     0, 0, 0, 0,  //0        Apparently, no parameter settings 
                  //         dedicated to this group are needed.
     0, 6,-5, 1,  //495542
     0, 8,-4, 4,  //1517
    11,19, 1,11   //53248
*/
//iteration:939
/* 10867 */
0,       0,       0,       0,  //          0
0,       189,     -165,    28, //     401859
7,       165,     -213,    121,//      31601
//263,     421,     -93 ,    353,//          0
        }
    };


//based on 77566 pairs (deutsch_affix_2464_3.UTF8.txt)
//iteration:83
/* 11434 */
 bestParms best_ge_suffix =
     {
     true,
     true,
     "ge",
     4,
//based on 77566 pairs (deutsch_affix_2464_3.UTF8.txt)
//iteration:77
/* 14206 */
        {
/*
    0, 3,-2, 2,//	408111
    0, 3, 0,-2,//	15424
    7, 5, 3, 1//	15449
*/
// After introduction of parmsoff:
//iteration:207
/* 13549 */
/*
     0, 3,  0, 3,//              290921
     0, 1, -1, 0,//              36836
     0, 5, -3, 2,//              15198
    -8,-6,-11, 1 //              12888
*/
//iteration:539
/* 13346 */
0,       0,       0,       0, //      0
0,       0,       0,       0, //      0
0,       21,      -19,     2, //      172352
-11,     25,      -25,     25,//      11892
         }
    };

bestParms best_la =
    {
    true,
    false,
    "la",
    3,
//OnlyZeros 12 
//suffix only no 
//iteration:19.50
/* 6429 */
        {
            1,  289, -273,   27, //210484
           16,  808, -408,  512, //6084
         -448,  384, -896,  320, //615
          256,  256,-1280, 1024  //0
                                 //        0
        }
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     no
cutoff          2
fraction        9856.000000
iterations      31
trainlines      17954.000000
rules           978.612903
rules%         5.450668
same%stdev     2.828105
ambi1%stdev    0.547465
ambi2%stdev    0.472805
ambi3%stdev    0.000000
diff%stdev     2.688214
same%          50.361830  (old lemmatiser: 49.540047, cutoff 1)
ambi1%         0.699129
ambi2%         0.870845
ambi3%         0.000000
diff%          48.068196
amb.rules%     2.845578
false_amb%     2.551208
false_not_amb% 6.328959
true_amb%      0.294370
true_not_amb%  90.825463
precision       0.054545
recall          0.044444
*/
    };

bestParms best_la_suffix =
    {
    true,
    true,
    "la",
    3,
//OnlyZeros 12 
//suffix only yes 
//iteration:19.61
/* 7929 */
        {
          368, 1051,  -20, -440, //305094
        -1416, -310,-1984,    8, //6613
         -832,  416,-1488,   16, //2386
          512,  512,-1280,  256  //0
                                 //        0
        }
/*
New algorithm, least wrongly lemmatised (MIN(diff)).
Suffix only     yes
cutoff          0
fraction        9856.000000
iterations      31
trainlines      17954.000000
rules           9680.612903
rules%         53.918976
same%stdev     3.264852
ambi1%stdev    0.585340
ambi2%stdev    0.825352
ambi3%stdev    0.000000
diff%stdev     3.120582
same%          46.363302  (old lemmatiser: 49.540047, cutoff 1)
ambi1%         1.275604
ambi2%         1.373727
ambi3%         0.000000
diff%          50.987367
amb.rules%     6.181774
false_amb%     5.458114
false_not_amb% 5.899669
true_amb%      0.723660
true_not_amb%  87.918558
precision       0.062171
recall          0.109259
*/
    };

bestParms best_el =
    {
    true,
    false,
    "el",
    3,
//iteration:19.59
/* 31634 */
        {
            0,  611, -655,  126, //1141579
            2,  356, -317,   75, //162502
           32,  576,-1024,   32, //2011
          256,  768, -896, 1024  //0
                                 //        0
        }
//OnlyZeros 12 
//suffix only no 
    };

bestParms best_el_suffix =
    {
    true,
    true,
    "el",
    3,
#if 0
//iteration:19.60
/* 35707 */
        {
            0,   30,  -32,    1, //872036
            1,   15,   -5,   11, //90168
           -9,  848,-1057,  404, //12291
        -2944,-1536,-2560, -128  //0
                                 //        0
        }
//OnlyZeros 12
//suffix only yes
#endif
//iteration:19.33
/* 62522 */
        {
            0,    0,    0,    0, //0
            0,  988, -929,  243, //1334941
            1,  549, -531,   72, //81200
        -1024,  128, -672, 1312  //0
                                 //        0
        }
//OnlyZeros 12 
//suffix only yes 

    };


static struct bestParms bests[] =    
    {best_da
    ,best_da_suffix
    ,best_en
    ,best_en_suffix
    ,best_fr
    ,best_fr_suffix
    ,best_ge
    ,best_ge_suffix
    ,best_is
    ,best_is_suffix
    ,best_la
    ,best_la_suffix
    ,best_el
    ,best_el_suffix
    ,best_ru
    ,best_ru_suffix
    ,best_nl
    ,best_nl_suffix
    ,best_sl
    ,best_sl_suffix
    ,best_sv
    ,best_sv_suffix
    };

int best[NPARMS]    = 
  {// Before a chain of sibling rules is created, there are N candidate rules.
   // For each sibling created, the number n of remaining candidates is
   // decremented.
  0,  0,  0,  0, // n >= exp(N,0.75)
  0,  0,  0,  0, // n >= exp(N,0.5)
  0,  0,  0,  0, // n >= exp(N,0.25)
  0,  0,  0,  0  // n <  exp(N,0.25)
  };

//int D[NPARMS]        = {0};

int R[] = // 64 elements
    {
    1, 0, 0, 0,
    0, 1, 0, 0,
    0, 0, 1, 0,
    0, 0, 0, 1,
    1,-1, 0, 0,
    1, 1, 0, 0,
    1, 0,-1, 0,
    1, 0, 1, 0,
    1, 0, 0, 1,
    1, 0, 0,-1,
    0, 1,-1, 0,
    0, 1, 1, 0,
    0, 1, 0,-1,
    0, 1, 0, 1,
    0, 0, 1,-1,
    0, 0, 1, 1
    };

void plus(int * dest, int * term,int cols)
    {
    for(int col = 0;col < cols;++col)
        dest[col] += term[col];
    }

void copy(int * dest,int * source,int cols)
    {
    for(int col = 0;col < cols;++col)
        dest[col] = source[col];
    }

void betterfound(int Nnodes,const char * extra,int swath,int iterations)
    {
    copy(best,parms,NPARMS);
    FILE * f = fopen(besttxt,"a");
    if(f)
        {
        fprintf(f,"//iteration:%d.%d\n",swath,iterations);
        fprintf(f,"/* %d */\n",Nnodes);
        for(int i = 0;i < NPARMS;++i)
            {
            fprintf(f,"%d",parms[i]);
            if(((i+1) % 4) == 0)
                if(i == NPARMS - 1)
                    fprintf(f,"\n");
                else
                    fprintf(f,",\n");
            else
                fprintf(f,",\t");
            }
        fclose(f);
        }
    }

int minparmsoff = 0;

void copybest()
    {
    copy(parms,best,NPARMS); // go on with best result so far.
    }

const char * besttxt;
const char * parmstxt = "parms.txt";
int OnlyZeros = NPARMS;

bool brown()
    {
    static int it = 0;
    FILE * f = fopen(parmstxt,"a");
    assert(f);
//    D[rand() % NPARMS] += (rand() & 1) ? 1 : -1;

    int i;
    int T = it;
    int R0 = (T % 16) * 4;       // row selector in R[]: 0 4 8 12 ... 60 0 4 8 12 ... 60 0 4 ...
    T /= 16;                     // strip lowest 4 bits (it / 16)
    //int P0 = (T % 4) * 4;        // row selector in parms. After 16 iterations, the next row is modified: 0 0 0 ... (16x) 4 4 ... (16x) 8 ... (16x) 12 ... (16x) 0 ..
    int P0 = 12 - (T % 4) * 4;        // row selector in parms. After 16 iterations, the previous row is modified: 12 12 12 ... (16x) 8 8 ... (16x) 4 ... (16x) 0 ... (16x) 12 ..
    T /= 4;                      // strip 2 bits (it / 64)
    int fac = (T & 1) ? -1 : 1;  // 1 1 1 ... (64x) -1 -1 -1 ... (64x) 1 ...
    T /= 2;                      // strip 1 bit (it / 128)
    if(T * 128 == it)            // it = 0 128 256 ...
        {                        // double all parms of the best parameter setting and start a new round
        for(i = 0;i < NPARMS;++i)
            best[i] *= 2;        
        }
    /* NOT a good idea.
    if((it % 16) == 0) // After handling one row,
        copy(parms,best,NPARMS); // go on with best result so far.
        */
    copybest();
    //copy(parms,best,NPARMS); // go on with best result so far.
    for(i = 0;i < 4;++i)
        parms[P0+i] += fac*R[R0+i]; // 4N <= R0+i <= 4N+3
    ++it;

    fclose(f);

    for(i = 0;i < NPARMS; ++i)
        {
        if(parms[i])
            {
            minparmsoff = i / 4;
            minparmsoff *= 4;
            break;
            }
        }
    return false;//OnlyZeros <= P0;
    }

bool init()
    {
    int x2 = 0;
    int i;
    for(i = 0;i < NPARMS;++i)
        {
        x2 += parms[i]*parms[i];
        }
    if(x2 == 0)
        {
        copy(parms,best,NPARMS);
        }
    return true;
    }

int pcnt[(NPARMS >> 2)+1] = {0,0,0,0,0};


void onlyZeros()
    {
    OnlyZeros = 0;
    for(unsigned int i = 0;i < sizeof(pcnt)/sizeof(pcnt[0]);++i)
        {
        if(pcnt[i] != 0)
            {
            OnlyZeros = (i+1) << 2;
            pcnt[i] = 0;
            }
        }
    FILE * f = fopen(parmstxt,"a");
    assert(f);
    fprintf(f,"//OnlyZeros %d \n",OnlyZeros);
    fprintf(f,"//suffix only %s \n",::suffixonly ? "yes" : "no");
    fclose(f);
    }


void printparms(int Nnodes)
    {
    int i;
    FILE * f = fopen(parmstxt,"a");
    assert(f);
    fprintf(f,"/* %d */\n",Nnodes);
    fprintf(f,"        {\n        ");
    for(i = 0;i < NPARMS;++i)
        {
        fprintf(f,"%5d",parms[i]);
        if(((i+1) % 4) == 0)
            {
            if(i == NPARMS - 1)
                fprintf(f,"  //%d\n        ",pcnt[i >> 2]);
            else
                fprintf(f,", //%d\n        ",pcnt[i >> 2]);
            }
        else
            fprintf(f,",");
        }
    fprintf(f,"                         //%9d\n",pcnt[i >> 2]);
    fprintf(f,"        }\n");
    fclose(f);
    }

int comp_parms(const vertex * a,const vertex * b)
    {
    //for(int o = 0;o < NPARMS;o += 4)
    if(  a->R__R != b->R__R
      || a->W__R != b->W__R
      || a->R__W != b->R__W
      || a->W__W != b->W__W
      )
        {
        int off = minparmsoff;
        if(off < parmsoff)
            off = parmsoff;
        for(int o = off;o < NPARMS;o += 4)
            {
            int A = parms[o]*a->R__R + parms[o+1]*a->W__R + parms[o+2]*a->R__W + parms[o+3]*a->W__W;
            int B = parms[o]*b->R__R + parms[o+1]*b->W__R + parms[o+2]*b->R__W + parms[o+3]*b->W__W;
            if(A != B)
                {
                ++pcnt[o >> 2]; // For counting the number of times the first, second, third or fourth condition has been used.
                // (Hypothesis: with parms as doubles the first condition is used and only in very special cases the second.
                //  Addendum: This hypothesis holds.)
                return A > B ? -1 : 1;
                }
            }
        ++pcnt[NPARMS >> 2];
        }
    return 0;
    }

int comp_parms_off(const vertex * a,const vertex * b)
    {
    if(  a->R__R != b->R__R
      || a->W__R != b->W__R
      || a->R__W != b->R__W
      || a->W__W != b->W__W
      )
        {
        //for(int o = 0;o < NPARMS;o += 4)
        int off = minparmsoff;
        if(off < parmsoff)
            off = parmsoff;
        for(int o = off;o < NPARMS;o += 4)
            {
            int A = parms[o]*a->R__R + parms[o+1]*a->W__R + parms[o+2]*a->R__W + parms[o+3]*a->W__W;
            int B = parms[o]*b->R__R + parms[o+1]*b->W__R + parms[o+2]*b->R__W + parms[o+3]*b->W__W;
            if(A != B)
                {
                ++pcnt[o >> 2]; // For counting the number of times the first, second, third or fourth condition has been used.
                // (Hypothesis: with parms as doubles the first condition is used and only in very special cases the second.
                //  Addendum: This hypothesis holds.)
                return A > B ? -1 : 1;
                }
            }
        ++pcnt[NPARMS >> 2];
        }
    return 0;
    }


static int nparms = 0;

int comp_parms0(const vertex * a,const vertex * b)
    {
    for(int o = 0;o < nparms;o += 4)
        {
        int A = parms[o]*a->R__R + parms[o+1]*a->W__R + parms[o+2]*a->R__W + parms[o+3]*a->W__W;
        int B = parms[o]*b->R__R + parms[o+1]*b->W__R + parms[o+2]*b->R__W + parms[o+3]*b->W__W;
        if(A != B)
            {
            return A > B ? -1 : 1;
            }
        }
    return 0;
    }


int comp_parms0_off(const vertex * a,const vertex * b)
    {
    int off = minparmsoff;
    if(off < parmsoff)
        off = parmsoff;
    for(int o = off;o < nparms;o += 4)
        {
        int A = parms[o]*a->R__R + parms[o+1]*a->W__R + parms[o+2]*a->R__W + parms[o+3]*a->W__W;
        int B = parms[o]*b->R__R + parms[o+1]*b->W__R + parms[o+2]*b->R__W + parms[o+3]*b->W__W;
        if(A != B)
            {
            return A > B ? -1 : 1;
            }
        }
    return 0;
    }



struct funcstruct
    {
    bool compute_parms;
    char * number;
    char * name;
    int (*comp)(const vertex * a,const vertex * b);
    };

struct funcstruct funcstructs[] =
    {
        {true,"0","parms",comp_parms_off}, //makeaffix.exe mydata.txt 0 affixrules XX 123 parms
#if _NA
        {false,"1","fairly_good",comp_fairly_good},
        {false,"2","even_better",comp_even_better},
        {false,"3","affiksFEW3",comp_affiksFEW3},
        {false,"4","affiksFEW",comp_affiksFEW},
        {false,"5","affiksFEW2",comp_affiksFEW2},
        {false,"6","fixNA",comp_fixNA},
        {false,"7","fruit",comp_fruit},
        {false,"8","ice",comp_ice},
        {false,"9","pisang",comp_pisang},
        {false,"10","kiwi",comp_kiwi},
        {false,"11","carrot",comp_carrot},
        {false,"12","peen",comp_peen},
        {false,"13","beet",comp_beet},
        {false,"14","sugar",comp_sugar},
        {false,"15","affiksFEW2org",comp_affiksFEW2org},
#endif
        {false,"16","honey",comp_honey},
        {false,"17","koud",comp_koud},
        {false,"18","parms0",comp_parms0},
        {false,"19","parmsoff",comp_parms0_off},
        {false,0,0,0}
    };


bool setCompetitionFunction(const char * functionname,const char * extra,bool simple)
    {
    int i;
    for(i = 0;funcstructs[i].number;++i)
        if(!strcmp(functionname,funcstructs[i].number) || !strcmp(functionname,funcstructs[i].name))
            {
            comp = funcstructs[i].comp;
            if(comp == comp_parms_off && simple)
                comp = comp_parms;
            if(VERBOSE)
                {
                if(comp == comp_parms)
                    {
                    printf("comp == comp_parms\n");
                    }
                else if(comp == comp_parms_off)
                    {
                    printf("comp == comp_parms_off\n");
                    }
                }
            compute_parms = funcstructs[i].compute_parms;
            if(comp == comp_parms0 || comp == comp_parms0_off)
                {
                /*
    bool useoffset;
    bool suffixonly;
    char * langbase;
    int rows;
    int val[NPARMS];

                */
                for(unsigned int j = 0;j < sizeof(bests)/sizeof(bests[0]);++j)
                    {
                    if(bests[j].suffixonly == ::suffixonly && !strncmp(bests[j].langbase,extra,strlen(bests[j].langbase)))
                        {
                        printf("bests[%d].suffixonly == [%s] bests[%d].langbase == [%s]\n",j,bests[j].suffixonly ? "true" : "false",j,bests[j].langbase);
                        if(bests[j].useoffset)
                            {
                            printf("comp = comp_parms0_off\n");
                            comp = comp_parms0_off;
                            }
                        else
                            {
                            printf("comp = comp_parms0\n");
                            comp = comp_parms0;
                            }
                        printf("bests[%d].rows == [%d]\n",j,bests[j].rows);
                        nparms = bests[j].rows * ROWPARMS;
                        for(int k = 0;k < nparms;++k)
                            parms[k] = bests[j].val[k];
                        break;
                        }
                    }
                if(nparms == 0)
                    {
                    fprintf(stderr,"No parameters defined for \"%s\"\nChoose one of:\n",extra);
                    for(unsigned int j = 0;j < sizeof(bests)/sizeof(bests[0]);++j)
                        {
                        fprintf(stderr,"\t%s %s\n",bests[j].langbase,bests[j].suffixonly ? "suffix":"affix");
                        }
                    fprintf(stderr,"Or find optimal parameters for %s and put these in comp.cpp.\n",extra);
                    getchar();
                    exit(-1);
                    }
                if(VERBOSE)
                    {
                    if(comp == comp_parms0)
                        printf("comp_parms0\n");
                    else
                        printf("comp_parms0_off\n");
                    }
                }
            for(i = 0;i < nparms; ++i)
                {
                if(parms[i])
                    {
                    minparmsoff = i / 4;
                    minparmsoff *= 4;
                    break;
                    }
                }
            if(VERBOSE)
                printf("minparmsoff = %d \n",minparmsoff);
            return true;
            }
    return false;
    }