Page 1 of 1

Static evaluation test posistions

Posted: Wed Nov 25, 2015 3:57 am
by zd3nik
I am wondering if anyone knows of a set of test positions that are basically quiet (no immediate win by tactics) and have a clear positional advantage for one side. Preferably with the winning side having <= material compared to the losing side.

I've done a few searches with no luck so far.

Re: Static evaluation test posistions

Posted: Wed Nov 25, 2015 4:55 am
by Ferdy
zd3nik wrote:I am wondering if anyone knows of a set of test positions that are basically quiet (no immediate win by tactics) and have a clear positional advantage for one side. Preferably with the winning side having <= material compared to the losing side.

I've done a few searches with no luck so far.
Maybe STS has. How many positions do you want? If this is more better use a script.

Re: Static evaluation test posistions

Posted: Thu Nov 26, 2015 5:04 am
by Ferdy
I parsed the sts 1 to 15 and analyze each pos with sf6 at 200ms/pos saving those pos if score >= 20 cp and if side is behind or equal in material.

There are many, here are sample output.

Code: Select all

1kr5/3n4/q3p2p/p2n2p1/PppB1P2/5BP1/1P2Q2P/3R2K1 w - - ce 169; acs 0; acd 11; AE "sf6"; c0 "Material equal or behind but with good search score";
1n5k/3q3p/pp1p2pB/5r2/1PP1Qp2/P6P/6P1/2R3K1 w - - ce 210; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";
1n6/4bk1r/1p2rp2/pP2pN1p/K1P1N2P/8/P5R1/3R4 w - - ce 253; acs 0; acd 13; AE "sf6"; c0 "Material equal or behind but with good search score";
1nr5/1k5r/p3pqp1/3p4/1P1P1PP1/R4N2/3Q1PK1/R7 w - - ce 292; acs 0; acd 11; AE "sf6"; c0 "Material equal or behind but with good search score";
1q4k1/5p1p/p1rprnp1/3R4/N1P1P3/1P6/P5PP/3Q1R1K w - - ce 123; acs 0; acd 14; AE "sf6"; c0 "Material equal or behind but with good search score";
1qr1k2r/1p2bp2/pBn1p3/P2pPbpp/5P2/2P1QBPP/1P1N3R/R4K2 b k - ce 132; acs 0; acd 13; AE "sf6"; c0 "Material equal or behind but with good search score";
1r1b2k1/2r2ppp/p1qp4/3R1NPP/1pn1PQB1/8/PPP3R1/1K6 w - - ce 216; acs 0; acd 11; AE "sf6"; c0 "Material equal or behind but with good search score";
1r1qk1nr/p3ppbp/3p2p1/1pp5/2bPP3/4B1P1/2PQNPBP/R2R2K1 w k - ce 39; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";
1r2n1rk/pP2q2p/P2p4/4pQ2/2P2p2/5B1P/3R1P1K/3R4 w - - ce 434; acs 0; acd 13; AE "sf6"; c0 "Material equal or behind but with good search score";
1r4k1/1rq2pp1/3b1nn1/pBpPp3/P1N4p/2PP1Q1P/6PB/2R2RK1 w - - ce 229; acs 0; acd 13; AE "sf6"; c0 "Material equal or behind but with good search score";
1r4k1/p1rqbp1p/b1p1p1p1/NpP1P3/3PB3/3Q2P1/P4P1P/3RR1K1 w - - ce 80; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";
1r6/R1nk1p2/1p4pp/pP1p1P2/P2P3P/5PN1/5K2/8 w - - ce 145; acs 0; acd 15; AE "sf6"; c0 "Material equal or behind but with good search score";
1rr3k1/1q3pp1/pnbQp2p/1p2P3/3B1P2/2PB4/P1P2RPP/R5K1 w - - ce 42; acs 0; acd 14; AE "sf6"; c0 "Material equal or behind but with good search score";
2b1r1k1/1p6/pQ1p1q1p/P2P3P/2P1pPpN/6P1/4R1K1/8 w - - ce 170; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";
2bq2k1/1pr3bp/1Qpr2p1/P2pNp2/3P1P1P/6P1/5PB1/1RR3K1 w - - ce 204; acs 0; acd 14; AE "sf6"; c0 "Material equal or behind but with good search score";
2kr4/ppqnbp1r/2n1p1p1/P2pP3/3P2P1/3BBN2/1P1Q1PP1/R4RK1 w - - ce 108; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";
2r1kb1r/1bqn1pp1/p3p3/1p2P1P1/3Np3/P1N1B3/1PP1Q2P/R4RK1 w k - ce 34; acs 0; acd 12; AE "sf6"; c0 "Material equal or behind but with good search score";

&#91;...&#93;
Just in case if you need it, here is the python script, it can take any epd or fen files, change engine setting at analyze_fen(). Required files are mentioned at top of source. Analysis engine is for uci engine only.

Did not check if bestmove of engine here is a capture that gains material.
I Just rely to sts i.e this is strategic test suite.

Code: Select all

"""
get_interesting_pos_from_epd.py
developed on python 2.7.6

Good Score Position Extractor

v1
1. Read epd, find and save pos with compensation for material
Required&#58;
1. inputfile.epd
2. sf6.exe

Output
out_inputfile.epd

"""

from __future__ import print_function
import sys
import os
import subprocess

# Const
INIT_SCORE = -1000000
INIT_DEPTH = 1000
APP_NAME = 'Good Score Position Extractor'
APP_VER = '1.0'


def side_is_equal_or_behind_in_material&#40;s, epd&#41;&#58;
    """ input is epd or fen
        s can be w or b
        returns true if side to move is behind or equal in material
    """

    Q = epd.count&#40;'Q')
    q = epd.count&#40;'q')
    R = epd.count&#40;'R')
    r = epd.count&#40;'r')
    B = epd.count&#40;'B')
    b = epd.count&#40;'r')
    N = epd.count&#40;'N')
    n = epd.count&#40;'n')
    P = epd.count&#40;'P')
    p = epd.count&#40;'p')

    wmat = Q * 10 + R * 5 + B * 3 + N * 3 + P * 1
    bmat = q * 10 + r * 5 + b * 3 + n * 3 + p * 1
    
    if s == 'w'&#58;
        if wmat <= bmat&#58;
            return True;
    else&#58;
        assert s == 'b'
        if bmat <= wmat&#58;
            return True

    return False

    
def delete_file&#40;file_to_delete&#41;&#58;
    """ Delete existing file """    
    if os.path.isfile&#40;file_to_delete&#41;&#58;
        os.remove&#40;file_to_delete&#41;
        

def analyze_fen&#40;engine, fen_list, inputfilename&#41;&#58;
    """
    Save pos with good score even if material is equal or behind
    fen_list&#58; a list with 4 epd fields
    """

    # Engine option
    hashval = 64  # 64mb
    threadsval = 1

    movetime = 200  # Use millisec, 1 sec = 1000 millisec
    
    pos_read_cnt = 0
    pos_saved_cnt = 0
    
    outputfilename = 'out_' + inputfilename
    delete_file&#40;outputfilename&#41;  # Delete existing file    
    
    AEngine = engine&#91;0&#58;-4&#93;

    # Start the engine    
    p = subprocess.Popen&#40;engine, stdin=subprocess.PIPE, stdout=subprocess.PIPE&#41;

    # Send uci command
    p.stdin.write&#40;'uci\n')
    
    for eline in iter&#40;p.stdout.readline, '')&#58;
        peline = eline.strip&#40;)
        print&#40;peline&#41;  # Print engine info at startup
        if 'uciok' in peline&#58;
            break;

    # Set option values
    p.stdin.write&#40;'setoption name Hash value %d\n' %&#40;hashval&#41;)
    p.stdin.write&#40;'setoption name Threads value %d\n' %&#40;threadsval&#41;)

    # Read fen and analyze
    for fens in fen_list&#58;

        # Init values for saving needed pos
        score = INIT_SCORE
        depth = INIT_DEPTH
        
        pos_read_cnt += 1

        # Console progress display
        sys.stdout.write&#40;'pos %d \r' %&#40;pos_read_cnt&#41;)

        # Prepare the engine to execute
        p.stdin.write&#40;"ucinewgame\n")
        fen = fens + ' 0 1'

        # We don't check if pos has legal moves or not
        p.stdin.write&#40;'position fen ' + fen + ' 0 1\n')
        p.stdin.write&#40;'go movetime %d\n' %&#40;movetime&#41;)

        # Parse engine output and get score and depth info
        for eline in iter&#40;p.stdout.readline, '')&#58;
            aa = eline.strip&#40;)
            # print&#40;aa&#41; # Print engine output
            
            bb = aa.split&#40;' ')
            if 'cp' in bb&#58;
                score = int&#40;bb&#91;bb.index&#40;'cp')+1&#93;)
            elif 'mate' in bb&#58;
                score = 6000
            if 'depth' in bb and 'score' in bb&#58;
                depth = int&#40;bb&#91;bb.index&#40;'depth')+1&#93;)
                
            if 'bestmove' in aa&#58;
                s = fen.split&#40;' ')
                s = s&#91;1&#93;  # Get side to move
                assert s == 'w' or s == 'b'
                if side_is_equal_or_behind_in_material&#40;s, fen&#41;&#58;
                    if score >= 20 and score != INIT_SCORE and depth != INIT_DEPTH&#58;
                        with open&#40;outputfilename, "a+") as fout&#58;
                            fout.write&#40;'%s ce %d; acs %d; acd %d; AE \"%s\"; c0 \"%s\";\n' %&#40;fens,
                                        score,
                                        movetime/1000,
                                        depth,
                                        AEngine,
                                        'Material equal or behind but with good search score'))
                            pos_saved_cnt += 1
                break
            
    # Quit engine when all fen are analyzed
    p.stdin.write&#40;'quit\n')
    p.stdin.close&#40;)
    p.communicate&#40;)

    print&#40;'Done!!, read_pos %d, saved_pos %d\n' %&#40;pos_read_cnt, pos_saved_cnt&#41;)
        

def main&#40;argv&#41;&#58;
    """ Main """
    input_fn = 'inputfile.epd'
    aengine = 'sf6.exe'

    # Check files
    if not os.path.isfile&#40;input_fn&#41;&#58;
        print&#40;'%s is missing!!' %&#40;input_fn&#41;)
        sys.exit&#40;1&#41;

    if not os.path.isfile&#40;aengine&#41;&#58;
        print&#40;'%s is missing!!' %&#40;aengine&#41;)
        sys.exit&#40;1&#41;

    fenlist = &#91;&#93;

    with open&#40;input_fn, 'r') as f&#58;
        for lines in f&#58;
            line = lines.strip&#40;)

            # Get the first 4 fields on fen or epd file
            s_epd = line.split&#40;' ')
            s_epd = s_epd&#91;0&#58;4&#93;
            s_epd = ' '.join&#40;s_epd&#41;

            fenlist.append&#40;s_epd&#41;  # Save to list

    # Analyze the saved epd list
    analyze_fen&#40;aengine, fenlist, input_fn&#41;
    

if __name__ == '__main__'&#58;
    main&#40;sys.argv&#91;1&#58;&#93;)

Re: Static evaluation test posistions

Posted: Fri Nov 27, 2015 6:14 am
by zd3nik
Ferdy, you have a very good idea here for finding the kind of positions I'm talking about. Some of the positions you found are probably a bit too deep for static analysis. But that's nothing a little tweaking of the python script couldn't solve. Thanks!

Re: Static evaluation test posistions

Posted: Fri Nov 27, 2015 6:52 am
by Ferdy
There is error in the script.

Change from (mistake), counting black bishop

Code: Select all

b = epd.count&#40;'r')
to (correct)

Code: Select all

b = epd.count&#40;'b')

Also from this one

Code: Select all

# Prepare the engine to execute 
p.stdin.write&#40;"ucinewgame\n") 
fen = fens + ' 0 1' 
The following is not necessary, because the 0 1 was added when the position fen ... is sent to the engine.

Code: Select all

fen = fens + ' 0 1'

Re: Static evaluation test posistions

Posted: Fri Nov 27, 2015 7:34 am
by Ferdy
I am also developing a script to take pgn and output interesting epd positions with different criteria. This is the one I will use to create training pos for eval auto-tuning. The epd has lots of info in it. Here is an example so far.

Code: Select all

r2qkbnr/1b1p1pp1/ppn1p3/2p4p/2P1P3/2NP1NP1/PP3PBP/R1BQ1RK1 b kq - sm Bd6; ce -35; eco "B28"; c0 "f8d6"; c1 "1-0"; c2 "40"; c3 "53";

Code: Select all

sm is the move in actual game, this can be used for move tuning.
ce is the actual move score by the engine in cp, this can be used to tune by score.
eco is to know which opening you will use to optimize your eval.
c0 is the equivalent uci move format, for direct comparison of uci engine output.
c1 is the result suitable for Texel method.
c2 is the material of white, q = 10, r = 5, minors = 3, and p = 1, for better control of the material config that will be considered in training pos.
c3 is the material for black.

Also the epd will be save only when the current move of the pos, and the move of the next pos, and a move in the next next pos are not tactical or special move i.e capture, promote, castle, and move that checks the opp king. This is a 3 ply look ahead without tactical move found in the actual game move. An idea from Miguel but he has I think 6 plies ahead quiet moves.
The purpose is for tuning the eval directly by calling the static_eval(), instead of calling the qsearch(). But of course this can also be used when using qsearch() score.

If you are interested on this I will send this to you once completed. The code was done and I am just testing the output now.

If interested prepare a pgn that is an output from cutechess-cli with move comments from ponder off games. This is my target pgn file to process.

Marcel mentioned of taking the elo difference, on those positions which is interesting to be included here but by default the pgn from cutechess-cli has no elo tags, although we can imbed it there but this will come in the next script with elo difference.

Adam and Daniel gave me some sample pgn, so I will upload some epd coming from some of those pgn files.

Perhaps the output from this tool will be what you need by tinkering the material opcode, I just can't understand when you say that it might be too deep for static analysis.

Re: Static evaluation test posistions (code v1.1)

Posted: Fri Nov 27, 2015 8:02 am
by Ferdy
New version with bug fixes.

Code: Select all

"""
get_interesting_pos_from_epd.py
developed on python 2.7.6

Good Score Position Extractor

v1.1
1. Fixed counting of material, black rook was counted as black bishop,
also the b in side to move and q in castling right was counted in the material
which were wrong.
2. Fixed "position fen epd 0 1 0 1" in fen when given to uci engine,
it should only be "position fen epd 0 1"

v1
1. Read epd, find and save pos with compensation for material
Required&#58;
1. inputfile.epd
2. sf6.exe

Output
out_inputfile.epd

"""

from __future__ import print_function
import sys
import os
import subprocess

# Const
INIT_SCORE = -1000000
INIT_DEPTH = 1000
APP_NAME = 'Good Score Position Extractor'
APP_VER = '1.1'


def side_is_equal_or_behind_in_material&#40;s, epd&#41;&#58;
    """ input is epd or fen
        s can be w or b
        returns true if side to move is behind or equal in material
    """

    # Take only the piece field
    nepd = epd.split&#40;)
    nepd = nepd&#91;0&#93;
    epd = nepd

    Q = epd.count&#40;'Q')
    q = epd.count&#40;'q')
    R = epd.count&#40;'R')
    r = epd.count&#40;'r')
    B = epd.count&#40;'B')
    b = epd.count&#40;'b')
    N = epd.count&#40;'N')
    n = epd.count&#40;'n')
    P = epd.count&#40;'P')
    p = epd.count&#40;'p')

    wmat = Q * 10 + R * 5 + B * 3 + N * 3 + P * 1
    bmat = q * 10 + r * 5 + b * 3 + n * 3 + p * 1
    
    if s == 'w'&#58;
        if wmat <= bmat&#58;
            return True;
    else&#58;
        assert s == 'b'
        if bmat <= wmat&#58;
            return True

    return False

    
def delete_file&#40;file_to_delete&#41;&#58;
    """ Delete existing file """    
    if os.path.isfile&#40;file_to_delete&#41;&#58;
        os.remove&#40;file_to_delete&#41;
        

def analyze_fen&#40;engine, fen_list, inputfilename&#41;&#58;
    """
    Save pos with good score even if material is equal or behind
    fen_list&#58; a list with 4 epd fields
    """

    # Engine option
    hashval = 64  # 64mb
    threadsval = 1

    movetime = 200  # Use millisec, 1 sec = 1000 millisec
    
    pos_read_cnt = 0
    pos_saved_cnt = 0
    
    outputfilename = 'out_' + inputfilename
    delete_file&#40;outputfilename&#41;  # Delete existing file    
    
    AEngine = engine&#91;0&#58;-4&#93;

    # Start the engine    
    p = subprocess.Popen&#40;engine, stdin=subprocess.PIPE, stdout=subprocess.PIPE&#41;

    # Send uci command
    p.stdin.write&#40;'uci\n')
    
    for eline in iter&#40;p.stdout.readline, '')&#58;
        peline = eline.strip&#40;)
        print&#40;peline&#41;  # Print engine info at startup
        if 'uciok' in peline&#58;
            break;

    # Set option values
    p.stdin.write&#40;'setoption name Hash value %d\n' %&#40;hashval&#41;)
    p.stdin.write&#40;'setoption name Threads value %d\n' %&#40;threadsval&#41;)

    # Read fen and analyze
    for fens in fen_list&#58;

        # Init values for saving needed pos
        score = INIT_SCORE
        depth = INIT_DEPTH
        
        pos_read_cnt += 1

        # Console progress display
        sys.stdout.write&#40;'pos %d \r' %&#40;pos_read_cnt&#41;)

        # Prepare the engine to execute
        p.stdin.write&#40;"ucinewgame\n")
        fen = fens

        # We don't check if pos has legal moves or not
        p.stdin.write&#40;'position fen ' + fen + ' 0 1\n')
        p.stdin.write&#40;'go movetime %d\n' %&#40;movetime&#41;)

        # Parse engine output and get score and depth info
        for eline in iter&#40;p.stdout.readline, '')&#58;
            aa = eline.strip&#40;)
            print&#40;aa&#41; # Print engine output
            
            bb = aa.split&#40;' ')
            if 'cp' in bb&#58;
                score = int&#40;bb&#91;bb.index&#40;'cp')+1&#93;)
            elif 'mate' in bb&#58;
                score = 6000
            if 'depth' in bb and 'score' in bb&#58;
                depth = int&#40;bb&#91;bb.index&#40;'depth')+1&#93;)
                
            if 'bestmove' in aa&#58;
                s = fen.split&#40;' ')
                s = s&#91;1&#93;  # Get side to move
                assert s == 'w' or s == 'b'
                if side_is_equal_or_behind_in_material&#40;s, fen&#41;&#58;
                    if score >= 20 and score != INIT_SCORE and depth != INIT_DEPTH&#58;
                        with open&#40;outputfilename, "a+") as fout&#58;
                            fout.write&#40;'%s ce %d; acs %d; acd %d; AE \"%s\"; c0 \"%s\";\n' %&#40;fens,
                                        score,
                                        movetime/1000,
                                        depth,
                                        AEngine,
                                        'Material equal or behind but with good search score'))
                            pos_saved_cnt += 1
                break
            
    # Quit engine when all fen are analyzed
    p.stdin.write&#40;'quit\n')
    p.stdin.close&#40;)
    p.communicate&#40;)

    print&#40;'Done!!, read_pos %d, saved_pos %d\n' %&#40;pos_read_cnt, pos_saved_cnt&#41;)
        

def main&#40;argv&#41;&#58;
    """ Main """
    input_fn = 'inputfile.epd'
    aengine = 'sf6.exe'

    # Check files
    if not os.path.isfile&#40;input_fn&#41;&#58;
        print&#40;'%s is missing!!' %&#40;input_fn&#41;)
        sys.exit&#40;1&#41;

    if not os.path.isfile&#40;aengine&#41;&#58;
        print&#40;'%s is missing!!' %&#40;aengine&#41;)
        sys.exit&#40;1&#41;

    fenlist = &#91;&#93;

    with open&#40;input_fn, 'r') as f&#58;
        for lines in f&#58;
            line = lines.strip&#40;)

            # Get the first 4 fields on fen or epd file
            s_epd = line.split&#40;' ')
            s_epd = s_epd&#91;0&#58;4&#93;
            s_epd = ' '.join&#40;s_epd&#41;

            fenlist.append&#40;s_epd&#41;  # Save to list

    # Analyze the saved epd list
    analyze_fen&#40;aengine, fenlist, input_fn&#41;
    

if __name__ == '__main__'&#58;
    main&#40;sys.argv&#91;1&#58;&#93;)