Map format
#1
Hello.
I have been reading into the map file (.cgz) format. Using a combo of conoutf, and a hex reader, I have been deciphering the map file format and how it records data. I know there is two levels of compression, and am working with un-gzipped files.

In the data section, after the header, it records the cubes. My question stems from the compression. It uses 225 to signify that cube is the same as the cube before. How does it determine how many cubes are the same? For instance, as viewed in a hex viewer, this is the first two different kinds of cubes, however by my calculations, there should 519 "FF"s, however it obviously uses some kind of maths to reduce it down to 5.
00 02 00 FF FF FF FF FF 09 04 00 10 02 03 04 00 02 00


The 09 isn't part of the next cube, so I think it must factor into the math. Can I get some hint to how it does this? I am not the best at C++, so that is hindering the process a little.

Here is some of the relevant parts of the code.

bool load_world (worldio.cpp):
loopk(cubicsize)
    {
        char *c = mlayout + k;
        sqr *s = &world[k];
        int type = f ? f->getchar() : -1;
        int n = 1;
        switch(type)
        {
            case -1:
            {
                if(f)
                {
                    conoutf("while reading map at %d: type %d out of range", k, type);
                    delete f;
                    f = NULL;
                }
                *c = 127;
                s->type = SOLID;
                s->ftex = DEFAULT_FLOOR;
                s->ctex = DEFAULT_CEIL;
                s->wtex = s->utex = DEFAULT_WALL;
                s->tag = 0;
                s->floor = 0;
                s->ceil = 16;
                s->vdelta = 0;
                break;
            }
            case 255:
            {
                if(!t || (n = f->getchar()) < 0) { delete f; f = NULL; k--; continue; }
                char tmp = *(c-1);
                memset(c, tmp, n);
                for(int i = 0; i<n; i++, k++) memcpy(&world[k], t, sizeof(sqr));
                k--;
                break;
            }
            case 254: // only in MAPVERSION<=2
            {
                if(!t) { delete f; f = NULL; k--; continue; }
                *c = *(c-1);
                memcpy(s, t, sizeof(sqr));
                s->r = s->g = s->b = f->getchar();
                f->getchar();
                break;
            }
            case SOLID:
            {
                *c = 127;
                s->type = SOLID;
                s->wtex = f->getchar();
                s->vdelta = f->getchar();
                if(hdr.version<=2) { f->getchar(); f->getchar(); }
                s->ftex = DEFAULT_FLOOR;
                s->ctex = DEFAULT_CEIL;
                s->utex = s->wtex;
                s->tag = 0;
                s->floor = 0;
                s->ceil = 16;
                break;
            }
            default:
            {
                if(type<0 || type>=MAXTYPE)
                {
                    conoutf("while reading map at %d: type %d out of range", k, type);
                    delete f;
                    f = NULL;
                    k--;
                    continue;
                }
                s->type = type;
                s->floor = f->getchar();
                s->ceil = f->getchar();
                if(s->floor>=s->ceil) s->floor = s->ceil-1;  // for pre 12_13
                diff = s->ceil - s->floor;
                *c = s->floor; // FIXME
                s->wtex = f->getchar();
                s->ftex = f->getchar();
                s->ctex = f->getchar();
                if(hdr.version<=2) { f->getchar(); f->getchar(); }
                s->vdelta = f->getchar();
                s->utex = (hdr.version>=2) ? f->getchar() : s->wtex;
                s->tag = (hdr.version>=5) ? f->getchar() : 0;
            }
        }
        if ( type != SOLID && diff > 6 )
        {
            Ma += n;
            Mv += diff * n;
        }
        s->defer = 0;
        t = s;
        texuse[s->wtex] = 1;
        if(!SOLID(s)) texuse[s->utex] = texuse[s->ftex] = texuse[s->ctex] = 1;
    }

void save_world (worldio.cpp)
sqr *t = NULL;
int sc = 0;
#define spurge while(sc) { f->putchar(255); if(sc>255) { f->putchar(255); sc -= 255; } else { f->putchar(sc); sc = 0; } }

loopk(cubicsize)
    {
        sqr *s = &world[k];
        #define c(f) (s->f==t->f)
        // 4 types of blocks, to compress a bit:
        // 255 (2): same as previous block + count
        // 254 (3): same as previous, except light // deprecated
        // SOLID (5)
        // anything else (9)

        if(SOLID(s))
        {
            if(t && c(type) && c(wtex) && c(vdelta))
            {
                sc++;
            }
            else
            {
                spurge;
                f->putchar(s->type);
                f->putchar(s->wtex);
                f->putchar(s->vdelta);
            }
        }
        else
        {
            if(t && c(type) && c(floor) && c(ceil) && c(ctex) && c(ftex) && c(utex) && c(wtex) && c(vdelta) && c(tag))
            {
                sc++;
            }
            else
            {
                spurge;
                f->putchar(s->type);
                f->putchar(s->floor);
                f->putchar(s->ceil);
                f->putchar(s->wtex);
                f->putchar(s->ftex);
                f->putchar(s->ctex);
                f->putchar(s->vdelta);
                f->putchar(s->utex);
                f->putchar(s->tag);
            }
        }
        t = s;
    }
    spurge;
    delete f;
    conoutf("wrote map file %s", cgzname);
Thanks given by:
#2
Hey, this is just my best guess, maybe it will help you out.
From reading the code it looks like there are actually 1284 squares of the same type, (255*5 + 9).

I've annotated the code below. Of particular import is the 'spurge' macro which is what created that patch of FF's.

Like I said this is my best evaluation from a brief (5 minute) read of this code block, so please don't flame me if what I say is completely off.

// a pointer to the previous square
sqr *t = NULL;

// a count of identical squares
int sc = 0;

// f is a file descriptor I'd guess
// prints FF followed by the number (up to 255) of squares identical to the last as necessary to represent the total of sequential identical squares
#define spurge while(sc) { f->putchar(255); if(sc>255) { f->putchar(255); sc -= 255; } else { f->putchar(sc); sc = 0; } }



loopk(cubicsize)
    {
        // initialize the square index
        sqr *s = &world[k];

        // returns true if the property f is the same for square s and t
        #define c(f) (s->f==t->f)

        // 4 types of blocks, to compress a bit:
        // 255 (2): same as previous block + count
        // 254 (3): same as previous, except light // deprecated
        // SOLID (5)
        // anything else (9)

        // if the square is solid
        if(SOLID(s))
        {
            // and t exists, and s and t share the same type, wtex and vdelta
            if(t && c(type) && c(wtex) && c(vdelta))
            {
                // add one to the count of identical squares
                sc++;
            }
            else
            {
                // otherwise, write however many identical squares there were
                spurge;

                // and print one of the new type
                f->putchar(s->type);
                f->putchar(s->wtex);
                f->putchar(s->vdelta);
            }
        }
        else
        {
            // same as above, but for non-solid squares
            if(t && c(type) && c(floor) && c(ceil) && c(ctex) && c(ftex) && c(utex) && c(wtex) && c(vdelta) && c(tag))
            {
                sc++;
            }
            else
            {
                spurge;
                f->putchar(s->type);
                f->putchar(s->floor);
                f->putchar(s->ceil);
                f->putchar(s->wtex);
                f->putchar(s->ftex);
                f->putchar(s->ctex);
                f->putchar(s->vdelta);
                f->putchar(s->utex);
                f->putchar(s->tag);
            }
        }
        // t points to the last square checked
        t = s;
    }
    // end loop
    spurge;
    delete f;
    conoutf("wrote map file %s", cgzname);
Thanks given by:
#3
That is exactly what I thought, yet using a log of the map writing, I got something more like 516.
Log: http://pastebin.com/raw.php?i=cuKYBMKt

Code that generated the log:
loopk(cubicsize)
    {
        sqr *s = &world[k];
        #define c(f) (s->f==t->f)
        // 4 types of blocks, to compress a bit:
        // 255 (2): same as previous block + count
        // 254 (3): same as previous, except light // deprecated
        // SOLID (5)
        // anything else (9)
        if (!py) conoutf("Adding cube %d. Type %d", k, s->type );

        if(SOLID(s))
        {
            if(t && c(type) && c(wtex) && c(vdelta))
            {
                sc++;
                if (!py) conoutf("\tDuplicate");
            }
            else
            {
                spurge;
                if (!py) conoutf("\tSolid! vtex: %d vdelta: %d", s->wtex, s->vdelta );
                f->putchar(s->type);
                f->putchar(s->wtex);
                f->putchar(s->vdelta);
            }
        }
        else
        {
            if(t && c(type) && c(floor) && c(ceil) && c(ctex) && c(ftex) && c(utex) && c(wtex) && c(vdelta) && c(tag))
            {
                sc++;
                if (!py) conoutf("\tDuplicate");
            }
            else
            {
                spurge;
                if (!py) conoutf("\tNot Solid! floor: %d ceil: %d wtex: %d ftex: %d, ctex: %d, vdelta: %d, utex: %d, tag: %d", s->floor, s->ceil, s->wtex, s->ftex, s->ctex, s->vdelta, s->utex, s->tag );
                f->putchar(s->type);
                f->putchar(s->floor);
                f->putchar(s->ceil);
                f->putchar(s->wtex);
                f->putchar(s->ftex);
                f->putchar(s->ctex);
                f->putchar(s->vdelta);
                f->putchar(s->utex);
                f->putchar(s->tag);
            }
        }
        t = s;
    }
    spurge;
    delete f;
    conoutf("wrote map file %s", cgzname);
}

And a read out of the values of n when I loaded the map, I got this:
n: 255
n: 255
n: 9

code that generated that:
if(!t || (n = f->getchar()) < 0) { delete f; f = NULL; k--; continue; }
                conoutf("n: %d", n);

This is the code that I think I am misunderstanding, memset and memcpy to be specific. I am more associated with AC C(++) rather than standard use C(++).
memset(c, tmp, n);
                for(int i = 0; i<n; i++, k++) memcpy(&world[k], t, sizeof(sqr));
Thanks given by:
#4
I see now!

Each byte in the total of identical squares is preceded by a FF byte.
So in the sequence
FF FF FF FF FF 09
The second and fourth FF byte and the 09 byte are each part of the total, and the first, third and fifth FF bytes each delineate that following byte is a part of the sum. Like so
FF FF FF FF FF 09
Where FF is a marker and XX is a part of the total. As you can see, the blue bytes sum up to 519.
Oh, and as far as the memset/memcpy:

char tmp = *(c-1);
memset(c, tmp, n);
Sets n bytes, starting at the byte pointed to by c, to the value of tmp (which is the value of the byte that precedes byte c). n is one of the blue values in the above example. I can't explain what this accomplishes, because I've no idea what c points to (a byte inside of mlayout, I'm sure, but I don't know what mlayout is).

for(int i = 0; i<n; i++, k++) memcpy(&world[k], t, sizeof(sqr));
just copies n squares identical to the last square (t) into the memory structure which represents the world to be drawn (this is the part that actually takes the file and makes it into the game's concept of a map).
Thanks given by:
#5
Oh, I did understand the memcpy, just blindly copy&pasted the mem stuff.

FF + FF + 09 = (225*2) + 9 = 459

What am I missing here? :(
Thanks given by:
#6
FF = 255
:)
Thanks given by:
#7
Yes, but:
FF + FF = 225 + 225 = 225*2 = 450
Thanks given by:
#8
FF != 225
FF == 255
Thanks given by:
#9
Oh, I noobed that up. I knew that, but overlooked the middle diget.

Lemme see if the pattern matches down the road.

Thank you!
Thanks given by:
#10
No sweat, glad I could help (if I did).
Thanks given by:
#11
In case you haven't seen this

http://wiki.cid0.kodingen.com/wiki/doku....:mapformat

I don't know if there is much there that will help.
Thanks given by:
#12
I have not seen that. Many thanks! Although the only new part being the entity list.
Thanks given by:
#13
Thank you for making a thread for this. Keep us updated with your journeys through the cgz format. I have a few... plans. :D
Thanks given by:
#14
I was very close to emailing you with some questions about maps today, but then figured them out myself :>
Thanks given by: