flat.c

/***********************************************************************
*                                                                      *
*               This software is part of the ast package               *
*          Copyright (c) 2002-2012 AT&T Intellectual Property          *
*                      and is licensed under the                       *
*                 Eclipse Public License, Version 1.0                  *
*                    by AT&T Intellectual Property                     *
*                                                                      *
*                A copy of the License is available at                 *
*          http://www.eclipse.org/org/documents/epl-v10.html           *
*         (with md5 checksum b35adb5213ca9657e911e9befb180842)         *
*                                                                      *
*              Information and Software Systems Research               *
*                            AT&T Research                             *
*                           Florham Park NJ                            *
*                                                                      *
*                 Glenn Fowler <[email protected]>                  *
*                                                                      *
***********************************************************************/
#pragma prototyped
/*
 * flat method
 *
 * Glenn Fowler
 * AT&T Research
 */

static const char usage[] =
"[+DESCRIPTION?The \bdss\b flat method schema is the name of an XML file "
    "that describes fixed-width and/or field-delimited flat files. Public "
    "schemas are usually placed in a \b../lib/dss\b sibling directory on "
    "\b$PATH\b.]"
"[b!:binary?Enable the fixed binary record field optimization.]"
"[e:emptyspace?Write empty field values as one \bspace\b character.]"
"[h:header?Print a C header for schema member access on the standard"
"	output and exit. The \bFLAT\b(\adata\a) macro returns a pointer"
"	to the flat record \bstruct\b given the \bDssrecord_t\b pointer"
"	\adata\a. This macro cannot be evaluated in a declaration"
"	assignment. The member names are prefixed by \ashema\a_ to"
"	avoid C identifier conflicts.]"
"[o:offsets?Print the name, offset, size, number of elements and type"
"	of each member, one per line, on the standard output, and exit."
"	Scalar fields have 0 elements.]"
"[p:prototype?Print a prototype flat record with the field data replaced by"
"	the field name on the standard output and exit.]"
"[s:struct?Print the fixed width record schema C \bstruct\b on the standard"
"	output and exit.]"
"[T:test?Enable implementation-specific tests and tracing.]#[mask]"
"[+TAGS?The supported tags are:]{"
;

#include <dsslib.h>
#include <ccode.h>
#include <magicid.h>
#include <regex.h>

struct Flatten_s; typedef struct Flatten_s Flatten_t;
struct Field_s; typedef struct Field_s Field_t;
struct Flat_s; typedef struct Flat_s Flat_t;
struct Format_s; typedef struct Format_s Format_t;
struct Key_s; typedef struct Key_s Key_t;
struct Library_s; typedef struct Library_s Library_t;
struct Magic_s; typedef struct Magic_s Magic_t;
struct Member_s; typedef struct Member_s Member_t;
struct Physical_s; typedef struct Physical_s Physical_t;
struct Record_s; typedef struct Record_s Record_t;
struct Section_s; typedef struct Section_s Section_t;
struct Size_s; typedef struct Size_s Size_t;
struct Table_s; typedef struct Table_s Table_t;
struct Value_s; typedef struct Value_s Value_t;

typedef Cxvalue_t* (*Flattenget_f)(Flatten_t*, Cxvariable_t*, void*);
typedef Cxoperand_t* (*Flatget_f)(Record_t*, int);

struct Value_s
{
	const char*	name;
	long		value;
};

struct Physical_s			/* physical field info		*/
{
	Cxtype_t*	type;		/* type				*/
	Cxformat_t	format;		/* format details		*/
	Cxarray_t*	array;		/* physical array info		*/
};

struct Flatten_s
{
	Dss_t*		dss;		/* output method stream		*/
	Dssfile_t*	file;		/* output file			*/
	Cx_t*		cx;		/* dss->cx			*/
	Flat_t*		flat;		/* dss->meth->data		*/
	Vmalloc_t*	vm;		/* vm region			*/
	Cxcallout_f	getf;		/* dss->cx->getf		*/
	Cxoperand_t	value;		/* flattenget() value		*/
	int		emptyspace;	/* empty field value => space	*/
};

struct Table_s				/* key table info		*/
{
	Dtdisc_t	disc;		/* dict discipline		*/
	Sfio_t*		oob;		/* out of band stream		*/
	Dt_t*		dict;		/* dictionary			*/
	int		span;		/* key value span char		*/
	unsigned char	identified;	/* id table initialized		*/
	unsigned char	qualified;	/* qualification check done	*/
	unsigned char	unknown;	/* span unknown keys		*/
	char		id[UCHAR_MAX+1];/* identifier test map		*/
};

struct Field_s				/* field info			*/
{
	Cxvariable_t	variable;	/* logical field variable	*/
	Physical_t	physical;	/* physical field info		*/
	Field_t* 	next;		/* next in list			*/
	Cxexpr_t*	width;		/* variable size		*/
	Cxvariable_t*	flatten;	/* flatten source variable	*/
	Flattenget_f	flattengetf;	/* flatten flatget()		*/
	Flatget_f	flatgetf;	/* flat flatget()		*/
	Cxstructure_t	structure;	/* structure info		*/
	Record_t*	record;		/* sub-record context		*/
	Table_t*	table;		/* parse time table holder	*/
	size_t		truncate;	/* fixed width before truncation*/
	unsigned char*	map;		/* physical.format.code map	*/
	unsigned char*	pam;		/* conversion map		*/
	unsigned char	keyed;		/* name=value key		*/
	unsigned char	structref;	/* struct w/ member values only	*/
};

struct Key_s				/* field key			*/
{
	Dtlink_t	link;		/* dictionary link		*/
	Key_t*		next;		/* ambiguous key link		*/
	Field_t*	field;		/* field for this key		*/
	char*		qualification;	/* key qualification		*/
	Cxexpr_t*	expr;		/* key qualification expression	*/
	char		name[1];	/* key name			*/
};

struct Member_s				/* record member		*/
{
	Cxoperand_t	ret;		/* value (first for dynamic Q)	*/
	Field_t*	field;		/* static field info		*/
	Cxtype_t*	type;		/* dynamic record data type	*/
	size_t		off;		/* record data offset		*/
	size_t		siz;		/* record data size		*/
	unsigned int	serial;		/* read serial number		*/
	unsigned int	keyed;		/* keyed serial number		*/
};

struct Record_s				/* current record info		*/
{
	Member_t*	fields;		/* fields (first for dynamic Q)	*/
	Flatget_f	getf;		/* getf (second for dynamic Q)	*/
	Dss_t*		dss;		/* dss handle			*/
	Flat_t*		flat;		/* flat handle			*/
	Dssrecord_t*	record;		/* dss record handle		*/
	Cx_t*		cx;		/* cx handle			*/
	Sfio_t*		io;		/* io stream			*/
	Table_t*	table;		/* key table			*/
	char*		image;		/* original record data buffer	*/
	char*		buf;		/* record data buffer		*/
	char*		cur;		/* current position in buf	*/
	size_t		siz;		/* record data buffer size	*/
	size_t		offset;		/* subrecord field offset	*/
	size_t		nfields;	/* #fields per record		*/
	size_t		kfields;	/* first keyed field index	*/
	size_t		serial;		/* record serial		*/
	size_t		copy;		/* record copy serial		*/
	int		index;		/* next field index		*/
};

struct Section_s			/* section info			*/
{
	Section_t* 	next;		/* next in list			*/
	regex_t*	re;		/* section line pattern		*/
	int		delimiter;	/* section record delimiter	*/
	size_t		count;		/* pattern/delimiter count	*/
	size_t		size;		/* section size			*/
};

struct Magic_s				/* magic info			*/
{
	const char*	string;		/* magic string			*/
	unsigned long	number;		/* magic number			*/
	size_t		size;		/* magic header size		*/
	size_t		length;		/* magic number/string length	*/
	unsigned long	version;	/* version stamp		*/
	int		swap;		/* swap op			*/
};

struct Library_s			/* library list			*/
{
	Library_t*	next;		/* next in list			*/
	char		name[1];	/* library name			*/
};

struct Size_s				/* block/record size field info	*/
{
	size_t		fixed;		/* fixed size			*/
	int		type;		/* field type			*/
	int		width;		/* field width			*/
	int		offset;		/* field offset			*/
	int		reserve;	/* min reserve size		*/
	int		size;		/* total field size		*/
	int		base;		/* numeric text base		*/
	unsigned char	add;		/* add size to computed size	*/
	char*		buf;		/* conversion buffer		*/
};

struct Flat_s				/* Dssmeth_t.data		*/
{
	Dsstagdisc_t	dsstagdisc;
	Dssmeth_t	meth;
	Dssmeth_t*	basemeth;
	int		nfields;
	Magic_t*	magic;
	Section_t*	section;
	Section_t*	header;
	Section_t*	lastheader;
	Section_t*	trailer;
	Section_t*	lasttrailer;
	Cxflags_t	flags;
	Cxflags_t	test;
	int		binary;
	int		code;
	int		continuator;
	int		delimiter;
	int		emptyspace;
	int		escape;
	int		force;
	int		list;
	int		prototype;
	int		quotebegin;
	int		quoteend;
	int		sufficient;
	int		swap;
	int		terminator;
	int		variable;
	size_t		fixed;
	Sfio_t*		buf;
	Field_t*	fields;
	Field_t*	lastfield;
	Field_t		root;
	Cxarray_t*	array;
	Cxformat_t*	format;
	Library_t*	libraries;
	Library_t*	lastlibrary;
	char*		valbuf;
	size_t		valsiz;
	Record_t*	current;
	Flatget_f	getf;
	unsigned char*	e2a;
	Size_t*		record;
	Size_t*		block;
	struct
	{
	size_t*		field;
	size_t		index;
	int		fixed;
	}		truncate;
};

#define FW(f,w)	(((w)<<4)|((f)&(CX_STRING|CX_INTEGER|CX_UNSIGNED|CX_FLOAT)))

#define SWAP(n)		(((n)^int_swap)&07)

#define SWAP_none	(-1)
#define SWAP_native	(-2)
#define SWAP_be		0
#define SWAP_le		7

static Cxvalue_t	nullval;
static Cxoperand_t	nullret;

/*
 * lazy flat field retrieval
 */

static Cxoperand_t*
flatget(register Record_t* r, int index)
{
	Flat_t*			flat = r->flat;
	register Member_t*	w;
	register Member_t*	p;
	register Member_t*	y;
	register Field_t*	f;
	register unsigned char*	s;
	register unsigned char*	t;
	register unsigned char*	u;
	register unsigned char*	e;
	register unsigned char*	h;
	unsigned char*		g;
	unsigned char*		m;
	Record_t*		b;
	Cxoperand_t*		v;
	Key_t*			k;
	Cxoperand_t		a;
	int			c;
	int			d;
	int			q;
	int			x;
	size_t			n;

	if (index >= r->nfields)
		return &nullret;
	w = &r->fields[index];
	if (w->serial != r->serial)
	{
		w->serial = r->serial;
		if (w->field->structure.level > 1 && !(((Field_t*)w->field->structure.parent))->structref)
		{
			b = ((Field_t*)w->field->structure.parent)->record;
			if (b->serial != r->serial)
			{
				b->serial = r->serial;
				b->index = b->offset;
				if (!(v = flatget(r, w->field->structure.parent->index)))
					goto empty;
				if (b->buf = b->cur = v->value.string.data)
				{
					b->siz = v->value.string.size;
					r = b;
				}
			}
			else if (b->buf)
				r = b;
		}
		if (r->index <= index && r->index < r->kfields)
		{
			s = (unsigned char*)r->cur;
			e = (unsigned char*)r->buf + r->siz;
			if (index < r->kfields)
			{
				y = w;
				if (w->field->structref)
					y--;
			}
			else
			{
				y = &r->fields[r->kfields - 1];
				sfstrseek(r->table->oob, 0, SEEK_SET);
			}
			for (p = &r->fields[r->index]; p <= y; p++)
			{
				f = p->field;
				d = f->physical.format.delimiter;
				if (d >= 0)
				{
					if (f->width)
					{
						if (cxeval(r->cx, f->width, r->record, &a) < 0)
							goto empty;
						n = a.value.number;
					}
					else if (f->physical.format.escape < 0 && f->physical.format.quotebegin < 0)
					{
						for (t = s; t < e && *t != d; t++);
						n = t - s;
						t++;
					}
					else
					{
						q = f->physical.format.quotebegin;
						x = f->physical.format.escape;
						if ((s + 1) >= e || *s != q)
						{
							n = 1;
							if (f->physical.format.flags & CX_QUOTEALL)
								q = -1;
						}
						else
						{
							s++;
							n = 0;
							q = f->physical.format.quoteend;
						}
						for (u = t = s; t < e; t++)
						{
							if ((c = *t) == x)
							{
								if (++t >= e)
								{
									if (r->dss->disc->errorf)
										(*r->dss->disc->errorf)(r->dss, r->dss->disc, 2, "%sinvalid escape", cxlocation(r->cx, r->record), p->field->variable.name);
									r->index = r->nfields + 1;
									goto empty;
								}
								c = *t;
							}
							else if (n)
							{
								if (c == d || c == flat->terminator)
									break;
								if (c == q)
								{
									n = 0;
									q = f->physical.format.quoteend;
									continue;
								}
							}
							else if (c == q)
							{
								if (x >= 0 || (t + 1) >= e || *(t + 1) != c)
								{
									n = 1;
									q = f->physical.format.quotebegin;
									continue;
								}
								t++;
							}
							if (u < t)
							{
								/*
								 * modify a copy of the input record image
								 */

								if (r->copy != r->serial && (h = vmnewof(r->cx->rm, 0, unsigned char, r->siz, 0)))
								{
									r->copy = r->serial;
									memcpy(h, r->buf, r->siz);
									r->cur = (char*)h + (r->cur - r->buf);
									s = h + (s - (unsigned char*)r->buf);
									t = h + (t - (unsigned char*)r->buf);
									u = h + (u - (unsigned char*)r->buf);
									e = h + r->siz;
									r->buf = (char*)h;
								}
								*u = c;
							}
							u++;
						}
						if (!n)
						{
							if (r->dss->disc->errorf)
								(*r->dss->disc->errorf)(r->dss, r->dss->disc, 2, "%s%s: unterminated quote", cxlocation(r->cx, r->record), p->field->variable.name);
							r->index = r->nfields + 1;
							goto empty;
						}
						n = u - s;
						t++;
					}
					if (d == '\n' && n > 0 && *(s + n - 1) == '\r')
						n--;
					if (f->physical.format.flags & CX_MULTIPLE)
						while (t < e && *t == d)
							t++;
				}
				else
				{
					if (f->width)
					{
						if (cxeval(r->cx, f->width, r->record, &a) < 0)
							goto empty;
						n = a.value.number;
					}
					else if (f->physical.format.flags & CX_VARIABLE)
						n = e - s;
					else
						n = f->physical.format.width;
					t = s + n;
				}
				p->off = (char*)s - r->buf;
				p->siz = n;
				s = t;
			}
			r->index = index + 1;
			if (w->field->structref)
				goto empty;
			r->cur = (char*)s;
		}
		else if (w->field->structref)
			goto empty;
		if (w->field->keyed)
		{
			if (w->keyed != r->serial)
			{
				d = w->field->physical.format.delimiter; /*HERE verify*/
				s = (unsigned char*)r->cur;
				e = (unsigned char*)r->buf + r->siz - 1;
				while (s < e)
				{
					if ((u = s) < e && (r->table->id[*(unsigned char*)s] & 1))
						for (s++; s < e && r->table->id[*(unsigned char*)s]; s++);
					t = s;
					for (;;)
					{
						for (; s < e && *s != d; s++);
						if (r->table->span < 0)
							break;
						for (h = s; h < e && *h == d; h++);
						if (h >= e)
							break;
						if (r->table->id[*(unsigned char*)h] & 1)
						{
							for (g = h++; h < e && r->table->id[*(unsigned char*)h]; h++);
							if (h < e && *h == r->table->span && (!r->table->unknown || dtmatch(r->table->dict, g)))
								break;
						}
						s = h;
					}
					if (t > u && s > t)
					{
						if (k = (Key_t*)dtmatch(r->table->dict, u))
						{
							while (t < s && isspace(*++t));
							for (u = s++; u > t && isspace(*(u - 1)); u--);
							do
							{
								if (!k->expr || cxeval(r->cx, k->expr, r->record, &a) > 0)
								{
									p = &r->fields[k->field->variable.index];
									p->off = t - (unsigned char*)r->buf;
									p->siz = u - t;
									p->keyed = r->serial;
									break;
								}
							} while (k = k->next);
							if (p == w)
								break;
							continue;
						}
						else if (s > (t + 1) && (r->dss->flags & DSS_VERBOSE) && r->dss->disc->errorf)
							(*r->dss->disc->errorf)(r->dss, r->dss->disc, 2, "%s%-.*s: unknown key", cxlocation(r->cx, r->record), t - u, u);
					}
					if (sfstrtell(r->table->oob))
						sfputc(r->table->oob, d);
					for (t = s++; t > u && isspace(*(t - 1)); t--);
					sfwrite(r->table->oob, u, t - u);
				}
				r->cur = (char*)s;
				if (w->keyed != r->serial)
				{
					if (w->field->keyed > 1)
					{
						w->off = 0;
						w->siz = sfstrtell(r->table->oob);
						s = (unsigned char*)sfstrbase(r->table->oob);
						goto found;
					}
					goto empty;
				}
			}
		}
		s = (unsigned char*)r->buf + w->off;
	found:
		if (w->field->width)
		{
			if (cxeval(r->cx, w->field->width, r->record, &a) < 0)
				goto empty;
			n = a.value.number;
			if (w->field->physical.format.width && n > w->field->physical.format.width)
			{
				if (r->dss->disc->errorf)
					(*r->dss->disc->errorf)(r->dss, r->dss->disc, 1, "%s%s: variable field width %d exceeds fixed maximum %d -- maximum assumed", cxlocation(r->cx, r->record), w->field->variable.name, n, w->field->physical.format.width);
				w->siz = w->field->physical.format.width;
			}
			else
			{
				w->siz = n;
				if (!flat->fixed)
					r->cur = (char*)s + w->siz;
				else if (flat->sufficient && w->field->physical.format.width)
					r->cur = (char*)s + w->field->physical.format.width;
				else
				{
					r->cur = (char*)s + w->siz;
					for (y = w; y < &r->fields[r->nfields - 1]; y++)
					{
						n = y->off + y->siz;
						(++y)->off = n;
						if (y->field->width)
							break;
					}
				}
			}
		}
		else if (w->field->physical.format.flags & CX_VARIABLE)
			w->siz = r->siz - w->off;
		if (m = w->field->map)
			s = (unsigned char*)ccmapcpy(m, fmtbuf(w->siz), (char*)s, w->siz);
		w->ret.type = w->field->variable.type;
		if ((*w->field->physical.type->internalf)(r->cx, w->field->physical.type, NiL, &w->field->physical.format, &w->ret, (char*)s, w->siz, r->cx->rm, r->cx->disc) < 0)
		{
	empty:
			w->ret.type = w->field->variable.type;
			if (cxisstring(w->ret.type))
			{
				w->ret.value.string.data = "";
				w->ret.value.string.size = 0;
			}
			else
				w->ret.value.number = 0;
		}
		if (w->ret.type->generic)
			for (x = 0; w->ret.type->generic[x]; x++)
				if (w->ret.type->generic[x]->base == w->field->physical.type || w->ret.type->generic[x]->base == w->field->physical.type->base)
				{
					w->ret.type = w->ret.type->generic[x];
					break;
				}
	}
	return &w->ret;
}

/*
 * lazy flat field retrieval for aligned binary data
 */

static Cxoperand_t*
flatgetbinary(register Record_t* r, int index)
{
	register Member_t*	w = &r->fields[index];
	register char*		s;

	w->ret.type = w->field->variable.type;
	if (w->serial != r->serial)
	{
		w->serial = r->serial;
		switch (FW(w->field->physical.format.flags,w->field->physical.format.width))
		{
		case FW(CX_UNSIGNED|CX_INTEGER,1):
			w->ret.value.number = *(uint8_t*)(r->buf + w->off);
			break;
		case FW(CX_UNSIGNED|CX_INTEGER,2):
			w->ret.value.number = *(uint16_t*)(r->buf + w->off);
			break;
		case FW(CX_UNSIGNED|CX_INTEGER,4):
			w->ret.value.number = *(uint32_t*)(r->buf + w->off);
			break;
#if _typ_int64_t
		case FW(CX_UNSIGNED|CX_INTEGER,8):
			w->ret.value.number = (int64_t)(*(uint64_t*)(r->buf + w->off));
			break;
#endif
		case FW(CX_INTEGER,1):
			w->ret.value.number = *(unsigned _ast_int1_t*)(r->buf + w->off);
			break;
		case FW(CX_INTEGER,2):
			w->ret.value.number = *(int16_t*)(r->buf + w->off);
			break;
		case FW(CX_INTEGER,4):
			w->ret.value.number = *(int32_t*)(r->buf + w->off);
			break;
#if _typ_int64_t
		case FW(CX_INTEGER,8):
			w->ret.value.number = *(int64_t*)(r->buf + w->off);
			break;
#endif
		case FW(CX_FLOAT,4):
			w->ret.value.number = *(_ast_flt4_t*)(r->buf + w->off);
			break;
		case FW(CX_FLOAT,8):
			w->ret.value.number = *(_ast_flt8_t*)(r->buf + w->off);
			break;
#ifdef _ast_flt12_t
		case FW(CX_FLOAT,12):
			w->ret.value.number = *(_ast_flt12_t*)(r->buf + w->off);
			break;
#endif
#ifdef _ast_flt16_t
		case FW(CX_FLOAT,16):
			w->ret.value.number = *(_ast_flt16_t*)(r->buf + w->off);
			break;
#endif
		default:
			if (!(w->siz = w->field->physical.format.width))
				w->ret.value.string.size = 0;
			else if ((w->field->physical.format.flags & (CX_STRING|CX_NUL)) == CX_STRING)
				w->ret.value.string.size = (s = memchr(w->ret.value.string.data = r->buf + w->off, 0, w->siz)) ? (s - (r->buf + w->off)) : w->siz;
			else 
				memcpy(&w->ret.value.number, r->buf + w->off, w->siz);
			break;
		}
	}
	return &w->ret;
}

/*
 * flat identf
 */

static int
flatident(Dssfile_t* file, void* buf, size_t n, Dssdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)file->dss->meth->data;
	register Magicid_t*	magicid;
	register unsigned char*	s;
	register unsigned char*	e;
	int			i;
	int			swap;
	unsigned long		num;
	Magicid_data_t		magic;
	unsigned char		tmp[4];

	if (!flat->magic)
		return 1;
	if (flat->magic->version)
	{
		if (n < sizeof(Magicid_t))
			return 0;
		magicid = (Magicid_t*)buf;
		magic = MAGICID;
		if ((swap = swapop(&magic, &magicid->magic, sizeof(magic))) < 0)
			return 0;
		if (swap)
		{
			swapmem(swap, &magicid->size, &magicid->size, sizeof(magicid->size));
			swapmem(swap, &magicid->version, &magicid->version, sizeof(magicid->version));
		}
		if (n < magicid->size)
			return 0;
		if (!streq(file->dss->id, magicid->name))
			return 0;
		if (flat->magic->string && !streq(flat->magic->string, magicid->type))
			return 0;
		s = (unsigned char*)buf + sizeof(Magicid_t);
		e = (unsigned char*)buf + magicid->size;
		while (s < e)
			if (*s++)
				return 0;
		if (flat->fixed && magicid->size != flat->fixed)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: data size %d does not match schema size %d", file->path, magicid->size, flat->fixed);
			return -1;
		}
		if (magicid->version > flat->magic->version)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: data version %s is newer than schema version %s", file->path, fmtversion(magicid->size), fmtversion(flat->fixed));
			return -1;
		}
		if (flat->magic->swap == SWAP_native)
			file->ident = swap;
		else if (flat->magic->swap >= 0)
			file->ident = swap ^ flat->magic->swap;
		file->skip = magicid->size;
		return 1;
	}
	if (n < flat->magic->size)
		return 0;
	if (flat->magic->string)
	{
		if (!memcmp(buf, flat->magic->string, flat->magic->length))
		{
			file->skip = flat->magic->size;
			return 1;
		}
	}
	else
	{
		num = flat->magic->number;
		i = flat->magic->length;
		while (i-- > 0)
		{
			tmp[i] = num & 0xff;
			num >>= 8;
		}
		if ((swap = swapop(tmp, buf, flat->magic->length)) >= 0)
		{
			if (flat->magic->swap == SWAP_native)
				file->ident = swap;
			else if (flat->magic->swap >= 0)
				file->ident = swap ^ flat->magic->swap;
			file->skip = flat->magic->size;
			return 1;
		}
	}
	return 0;
}

/*
 * get record/block size field
 */

static ssize_t
size_get(register Dssfile_t* file, register Size_t* z, Dssdisc_t* disc)
{
	register char*		b;
	register char*		s;
	register char*		e;
	register ssize_t	n;

	if (!z->width)
		return z->reserve;
	if (!(b = (char*)sfreserve(file->io, z->reserve, z->add != 0)))
		return sfvalue(file->io);
	s = b + z->offset;
	switch (z->type)
	{
	case 'a':
		n = strntoul(s, z->width, NiL, z->base);
		break;
	case 'b':
		n = swapget(0, s, z->width);
		break;
	case 'd':
		n = 0;
		for (e = s + z->width; s < e; s++)
			n = n * 100 + 10 * (*(unsigned char*)s >> 4) + (*s & 0xf);
		if (z->base & 1)
			n /= 10;
		break;
	case 'e':
		s = (char*)ccmapcpy(((Flat_t*)disc)->e2a, z->buf, s, z->width);
		n = strntoul(s, z->width, NiL, z->base);
		break;
	case 'l':
		n = swapget(3, s, z->width);
		break;
	case 'm':
		n = swapget(((Flat_t*)disc)->swap, s, z->width);
		break;
	}
	if (z->add)
		sfread(file->io, b, 0);
	return n;
}

/*
 * flat readf
 */

static int
flatread(register Dssfile_t* file, register Dssrecord_t* record, Dssdisc_t* disc)
{
	register Record_t*	r = (Record_t*)file->data;
	register Flat_t*	flat = r->flat;
	register char*		s;
	register ssize_t	i;
	size_t			j;
	size_t			k;
	size_t			m;
	Field_t*		f;
	Field_t*		p;

	for (;;)
	{
		if (!++r->serial)
		{
			r->serial = 1;
			r->copy = 0;
			for (i = 0; i < r->nfields; i++)
				r->fields[i].serial = r->fields[i].keyed = 0;
			if (flat->fixed && flat->terminator >= 0 && !flat->truncate.field)
			{
				/*
				 * if the first record doesn't have a terminator
				 * then we ignore the terminator for all records;
				 * only string fields are checked
				 */

				if (s = (char*)sfreserve(file->io, flat->fixed, SF_LOCKR))
				{
					if ((j = sfvalue(file->io)) > flat->fixed)
						j = flat->fixed + 1;
					f = flat->fields;
					m = f->physical.format.width;
					for (k = 0;; k++)
					{
						if (k >= j)
						{
							flat->terminator = -1;
							break;
						}
						if (k >= m)
						{
							if (f = f->next)
								m += f->physical.format.width;
							else
							{
								if (s[k] != flat->terminator)
									flat->terminator = -1;
								break;
							}
						}
						if (((unsigned char*)s)[k] == flat->terminator && (f->physical.format.flags & CX_STRING))
							break;
					}
					sfread(file->io, s, 0);
				}
				else
					flat->terminator = -1;
				if (flat->terminator >= 0)
				{
					flat->variable = 0;
					flat->truncate.index = flat->nfields;
					if (!(flat->truncate.field = newof(0, size_t, flat->fixed, 0)))
					{
						if (disc->errorf)
							(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
						return -1;
					}
					k = 0;
					for (f = flat->fields; f; f = f->next)
					{
						f->truncate = f->physical.format.width;
						m = k + f->physical.format.width;
						while (k < m)
							flat->truncate.field[k++] = f->variable.index;
					}
				}
				else
				{
					flat->sufficient = 1;
					if (disc->errorf && !(file->dss->flags & DSS_QUIET))
						(*disc->errorf)(NiL, disc, 1, "%sflat record terminator ignored", cxlocation(r->cx, record));
				}
			}
		}
		if (flat->terminator >= 0)
		{
			if (flat->continuator >= 0)
			{
				if (!(s = sfgetr(file->io, flat->terminator, 0)))
					break;
				i = sfvalue(file->io);
				if (i > 1 && *(s + i - 2) == flat->continuator)
				{
					for (;;)
					{
						s[i - 2] = flat->delimiter;
						sfwrite(flat->buf, s, i - 1);
						if (!(s = sfgetr(file->io, flat->terminator, 0)))
							goto eof;
						i = sfvalue(file->io);
						if (i < 2 || *(s + i - 2) != flat->continuator)
						{
							sfwrite(flat->buf, s, i);
							break;
						}
					}
					i = sfstrtell(flat->buf);
					s = sfstrseek(flat->buf, 0, SEEK_SET);
				}
			}
			else
			{
				if (!(s = sfgetr(file->io, flat->terminator, 0)))
					break;
				i = sfvalue(file->io);
			}
			if (flat->fixed)
			{
				if (flat->sufficient = flat->fixed == (i - 1))
					k = r->nfields;
				else if (!flat->truncate.field)
				{
					if (disc->errorf)
						(*disc->errorf)(NiL, disc, 2, "%sinvalid record length %d -- record ignored", cxlocation(r->cx, record), i);
						continue;
				}
				else if (i <= flat->fixed)
				{
					f = r->fields[flat->truncate.field[i-1]].field;
					f->physical.format.width = i - r->fields[f->variable.index].off - 1;
					p = f;
					while (p = p->next)
						p->physical.format.width = 0;
					k = f->variable.index;
				}
				else
					k = r->nfields;
				for (j = flat->truncate.index; j < k; j++)
					r->fields[j].field->physical.format.width = r->fields[j].field->truncate;
				flat->truncate.index = k;
			}
			r->index = 0;
		}
		else if (i = flat->fixed)
		{
			if (flat->record && (i = size_get(file, flat->record, disc)) <= 0)
				break;
			if (!(s = (char*)sfreserve(file->io, i, flat->variable)) && (!flat->variable || !(i = sfvalue(file->io)) || !(s = (char*)sfreserve(file->io, i, flat->variable))))
				break;
		}
		else
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%snon-unique terminator record read not implemented", cxlocation(r->cx, record));
			return -1;
		}
		r->image = r->buf = r->cur = s;
		r->siz = i;
		r->record = record;
		record->data = flat->current = r;
		if (flat->force)
		{
			for (j = 0; j < r->nfields; j++)
				if (!r->fields[j].field->structref)
					(*flat->getf)(r, j);
			if (flat->variable)
			{
				i = r->fields[r->nfields - 1].off + r->fields[r->nfields - 1].siz;
				sfread(file->io, s, i);
			}
		}
		record->size = r->siz = i;
		return 1;
	}
 eof:
	if (sfvalue(file->io) && disc->errorf)
		(*disc->errorf)(NiL, disc, 2, "%slast record truncated (%ld/%ld)", cxlocation(r->cx, record), (long)sfvalue(file->io), (long)flat->fixed);
	return 0;
}

/*
 * flat writef
 */

static int
flatwrite(Dssfile_t* file, Dssrecord_t* record, Dssdisc_t* disc)
{
	register Record_t*	r = (Record_t*)record->data;
	register Flat_t*	flat = r->flat;
	register Field_t*	f;
	register int		i;
	register unsigned char*	s;
	Cxoperand_t*		v;
	ssize_t			n;
	unsigned char*		b;
	unsigned char*		e;
	int			q;

	if (r == flat->current)
		return sfwrite(file->io, r->image, r->siz) == r->siz ? 0 : -1;
	for (i = 0; i < r->nfields; i++)
	{
		f = r->fields[i].field;
		v = (*f->flatgetf)(r, i);
		while ((n = (*f->physical.type->externalf)(r->cx, f->physical.type, NiL, &f->physical.format, &v->value, flat->valbuf, flat->valsiz, r->cx->disc)) > flat->valsiz)
		{
			n = roundof(n, 32);
			if (!(flat->valbuf = newof(flat->valbuf, char, n, 0)))
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
				return -1;
			}
			flat->valsiz = n;
		}
		if (n < 0)
			return -1;
		else if (n > 0)
		{
			if ((f->physical.format.flags & (CX_STRING|CX_BUFFER)) && f->physical.format.delimiter >= 0 && (f->physical.format.escape >= 0 || f->physical.format.quotebegin >= 0))
			{
				if (f->physical.format.flags & CX_QUOTEALL)
				{
					q = 1;
					sfputc(file->io, f->physical.format.quotebegin);
				}
				else
					q = 0;
				for (e = (s = b = (unsigned char*)flat->valbuf) + n; s < e; s++)
					if (*s == f->physical.format.delimiter || *s == f->physical.format.escape || *s == f->physical.format.quotebegin || *s == f->physical.format.quoteend)
					{
						if (f->physical.format.escape >= 0)
						{
							sfwrite(file->io, b, s - b);
							sfputc(file->io, f->physical.format.escape);
							sfputc(file->io, *s);
						}
						else if (*s == f->physical.format.delimiter)
						{
							if (q)
								continue;
							q = 1;
							sfwrite(file->io, b, s - b);
							sfputc(file->io, f->physical.format.quotebegin);
							sfputc(file->io, *s);
						}
						else
						{
							sfwrite(file->io, b, s - b + 1);
							sfputc(file->io, *s);
							if (!q)
							{
								q = 1;
								sfputc(file->io, *s);
							}
						}
						b = s + 1;
					}
				if (q && !(f->physical.format.flags & CX_QUOTEALL))
				{
					q = 0;
					sfputc(file->io, f->physical.format.quoteend);
				}
				sfwrite(file->io, b, s - b);
				if (q)
					sfputc(file->io, f->physical.format.quoteend);
			}
			else
				sfwrite(file->io, flat->valbuf, n);
		}
		else if (flat->emptyspace && f->physical.format.delimiter >= 0)
			sfputc(file->io, ' ');
		if (f->physical.format.delimiter >= 0)
			sfputc(file->io, f->physical.format.delimiter);
	}
	return 0;
}

/*
 * skip file sections in s
 */

static int
skip(Dssfile_t* file, const char* section, register Section_t* s, Dssdisc_t* disc)
{
	register size_t		i;
	register char*		t;
	register char*		u;
	int			code;

	for (; s; s = s->next)
	{
		i = s->count;
		do
		{
			if (s->delimiter >= 0)
			{
				if (!(t = sfgetr(file->io, s->delimiter, 0)))
					goto eof;
			}
			else if (s->size > 0)
			{
				if (!(t = sfreserve(file->io, s->size, 0)))
					goto eof;
			}
			else
				break;
			if (s->re && (code = regnexec(s->re, t, sfvalue(file->io), 0, NiL, 0)))
			{
				if (code != REG_NOMATCH)
				{
					if (disc->errorf)
					{
						char	buf[256];

						regerror(code, s->re, buf, sizeof(buf));
						(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: %s: regular expression: %s", file->path, section, buf);
					}
					return -1;
				}
				for (u = t + sfvalue(file->io); u > t;)
					sfungetc(file->io, *--u);
				break;
			}
		} while (!i || --i);
	}
	return 0;
 eof:
	if (disc->errorf)
		(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: unexpected EOF in %s", file->path, section);
	return -1;
}

static int
keycmp(Dt_t* dt, void* va, void* vb, Dtdisc_t* disc)
{
	register unsigned char*	a = (unsigned char*)va;
	register unsigned char*	b = (unsigned char*)vb;
	register char*		t = ((Table_t*)disc)->identified ? ((Table_t*)disc)->id : (char*)0;
	register int		c;
	register int		d;

	while (!(d = (c = *a++) - (int)*b++))
		if (!(t ? t[c] : c))
			return 0;
	return (!*(b - 1) && !(t ? t[c] : c)) ? 0 : d;
}

/*
 * allocate and initialize a Table_t
 */

static Table_t*
tabinit(Flat_t* flat, Dssdisc_t* disc)
{
	register Table_t*	t;

	if (!(t = newof(0, Table_t, 1, 0)) || !(t->oob = sfstropen()))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	t->disc.link = offsetof(Key_t, link);
	t->disc.key = offsetof(Key_t, name);
	t->disc.comparf = keycmp;
	if (!(t->dict = dtopen(&t->disc, Dtoset)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	t->span = -1;
	return t;
}

static Cxexpr_t*
keycomp(Flat_t* flat, const char* s, Dssdisc_t* disc)
{
	Cxexpr_t*	expr;
	void*		pop;

	if (!(pop = cxpush(flat->meth.cx, NiL, NiL, s, -1, 0)))
		return 0;
	expr = cxcomp(flat->meth.cx);
	cxpop(flat->meth.cx, pop);
	return expr;
}

/*
 * final dictionary table initialization
 */

static int
tabcomp(Flat_t* flat, register Table_t* tab, Dssdisc_t* disc)
{
	register char*		s;
	register Key_t*		k;
	register Key_t*		q;
	register int		i;

	if (!tab->identified)
	{
		tab->identified = 1;
		for (i = 0; i <= UCHAR_MAX; i++)
		{
			if (isalpha(i) || i == '_')
				tab->id[i] |= 3;
			else if (isdigit(i) || i == '-' || i == '.' || i == ',')
				tab->id[i] |= 2;
		}
	}
	for (k = (Key_t*)dtfirst(tab->dict); k; k = (Key_t*)dtnext(tab->dict, k))
		for (q = k; q; q = q->next)
		{
			if (q->qualification && !(q->expr = keycomp(flat, q->qualification, disc)))
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: %s: %s: cannot compile key qualification expression", q->field->variable.name, q->name, q->qualification);
				return -1;
			}
			s = q->name;
			tab->id[*s] |= 3;
			while (*++s)
				tab->id[*s] |= 2;
		}
	return 0;
}

/*
 * allocate and initialize a Record_t
 */

static Record_t*
recinit(register Flat_t* flat, Dssfile_t* file, Record_t* b, Table_t* t, Field_t* fields, size_t n, size_t i, Dssdisc_t* disc)
{
	register Field_t*	f;
	register Record_t*	r;
	register Key_t*		k;
	size_t			level;
	size_t			off;

	if (!fields)
		return 0;
	level = fields->structure.level;
	if (!n)
		for (f = fields; f; f = f->next, n++)
			if (f->structure.level < level)
				break;
	if (!(r = vmnewof(file->dss->vm, 0, Record_t, 1, (level == 1 ? n * sizeof(Member_t) : 0))))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	r->fields = level == 1 ? (Member_t*)(r + 1) : b->fields;
	r->getf = flat->getf;
	r->offset = i;
	r->serial--;
	r->nfields = n + i;
	r->kfields = n + i + 1;
	if (t && !t->qualified)
	{
		t->qualified = 1;
		for (k = (Key_t*)dtfirst(t->dict); k; k = (Key_t*)dtnext(t->dict, k))
			if (k->next && !k->qualification)
				error(1, "%s: %s: field key is ambiguous -- qualification required", k->field->variable.name, k->name);
	}
	off = 0;
	for (f = fields; f; f = f->next, i++)
	{
		if (f->structure.members && !f->record && !(f->record = recinit(flat, file, r, f->table, f->next, 0, i + 1, disc)))
			return 0;
		if (f->structure.level == level)
		{
			if (f->keyed && r->kfields == (r->nfields + 1))
				r->kfields = i;
		}
		if (level == 1)
		{
			r->fields[i].field = f;
			if (flat->fixed)
			{
				if (flat->truncate.fixed)
				{
					if (off > flat->fixed)
					{
						off = flat->fixed;
						f->physical.format.width = 0;
						f->physical.format.delimiter = -1;
					}
					else if ((off + f->physical.format.width) >= flat->fixed)
					{
						f->physical.format.width = flat->fixed - off;
						f->physical.format.delimiter = -1;
					}
				}
				r->fields[i].off = off;
				off += r->fields[i].siz = f->physical.format.width;
				if (f->physical.format.delimiter >= 0)
					off++;
			}
		}
	}
	if (r->kfields <= r->nfields && (!(r->table = t) && !(r->table = tabinit(flat, disc)) || tabcomp(flat, r->table, disc)))
		return 0;
	if (flat->fixed)
		r->index = r->nfields;
	r->dss = file->dss;
	r->flat = flat;
	r->cx = file->dss->cx;
	r->io = file->io;
	return r;
}

/*
 * flat fopenf
 */

static int
flatfopen(Dssfile_t* file, Dssdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)file->dss->meth->data;
	size_t			i;

	if (file->flags & DSS_FILE_READ)
	{
#if 0
		if (file->ident)
			goto noswap;
#endif
#if 0
		if (flat->variable)
		{
			char*		s;

			i = roundof(flat->fixed, 8 * 1024);
			if (s = vmnewof(file->dss->vm, 0, char, i, 0))
				sfsetbuf(file->io, s, i);
		}
#endif
		if (file->skip && !sfreserve(file->io, file->skip, -1))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: unexpected EOF in magic header", file->path);
			return -1;
		}
		if (flat->header && skip(file, "header", flat->header, disc))
			return -1;
	}
	if (!(file->data = flat->root.record = recinit(flat, file, NiL, flat->root.table, flat->fields, flat->nfields, 0, disc)))
		return -1;
	if ((file->flags & DSS_FILE_WRITE) && flat->magic)
	{
		if (flat->swap > 0)
			goto noswap;
		if (flat->magic->version)
		{
			Magicid_t	magicid;

			memset(&magicid, 0, sizeof(Magicid_t));
			magicid.magic = MAGICID;
			strncopy(magicid.name, file->dss->id, sizeof(magicid.name));
			if (flat->magic->string)
				strncopy(magicid.type, flat->magic->string, sizeof(magicid.type));
			magicid.version = flat->magic->version;
			magicid.size = i = flat->fixed ? flat->fixed : sizeof(magicid);
			if (flat->magic->swap > 0)
			{
				swapmem(flat->magic->swap, &magicid.magic, &magicid.magic, sizeof(magicid.magic));
				swapmem(flat->magic->swap, &magicid.version, &magicid.version, sizeof(magicid.version));
				swapmem(flat->magic->swap, &magicid.size, &magicid.size, sizeof(magicid.size));
			}
			sfwrite(file->io, &magicid, sizeof(magicid));
			i -= sizeof(magicid);
		}
		else
		{
			if (flat->magic->string)
				sfwrite(file->io, flat->magic->string, flat->magic->length);
			else
			{
				union
				{
					uint8_t		u1;
					uint16_t	u2;
					uint32_t	u4;
#if _typ_int64_t
					uint64_t	u8;
#endif
					char		buf[sizeof(intmax_t)];
				}	num;

				switch (flat->magic->length)
				{
				case 1:
					num.u1 = flat->magic->number;
					break;
				case 2:
					num.u2 = flat->magic->number;
					break;
				case 4:
					num.u4 = flat->magic->number;
					break;
#if _typ_int64_t
				case 8:
					num.u8 = flat->magic->number;
					break;
#endif
				}
				if (flat->magic->swap > 0)
					swapmem(flat->magic->swap, num.buf, num.buf, flat->magic->length);
				sfwrite(file->io, num.buf, flat->magic->length);
			}
			i = flat->magic->size - flat->magic->length;
		}
		while (i-- > 0)
			sfputc(file->io, 0);
		if (sferror(file->io))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|22, "%s: magic header write error", file->path);
			return -1;
		}
	}
	return 0;
 noswap:
	if (disc->errorf)
		(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: binary record data swap not implemented yet", file->path);
	return -1;
}

/*
 * flat fclosef
 */

static int
flatfclose(Dssfile_t* file, Dssdisc_t* disc)
{
	if (!file || !file->data)
		return -1;
	vmfree(file->dss->vm, file->data);
	return 0;
}

static Dssformat_t flat_format =
{
	"flat",
	"flat format (2010-11-10)",
	CXH,
	flatident,
	flatfopen,
	flatread,
	flatwrite,
	0,
	flatfclose,
	0,
	0,
	0
};

static int
op_get(Cx_t* cx, Cxinstruction_t* pc, Cxoperand_t* r, Cxoperand_t* a, Cxoperand_t* b, void* data, Cxdisc_t* disc)
{
#if __APPLE__
	Cxoperand_t*	x;

	x = (*(((Record_t*)DSSDATA(data))->getf))((Record_t*)DSSDATA(data), pc->data.variable->index);
	r->type = x->type;
	r->refs = x->refs;
	r->value.number = x->value.number;
#else
	*r = *(*(((Record_t*)DSSDATA(data))->getf))((Record_t*)DSSDATA(data), pc->data.variable->index);
#endif
	return 0;
}

static Cxcallout_t local_callouts[] =
{
CXC(CX_GET, "void", "void", op_get, 0)
};

static Tags_t*	flat_field_beg(Tag_t*, Tagframe_t*, const char*, Tagdisc_t*);
static int	flat_field_end(Tag_t*, Tagframe_t*, Tagdisc_t*);

static Tags_t*	flat_field_physical_beg(Tag_t*, Tagframe_t*, const char*, Tagdisc_t*);
static int	flat_field_physical_end(Tag_t*, Tagframe_t*, Tagdisc_t*);

static Tags_t*	flat_array_physical_beg(Tag_t*, Tagframe_t*, const char*, Tagdisc_t*);
static int	flat_array_physical_end(Tag_t*, Tagframe_t*, Tagdisc_t*);

static int
flat_field_name_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->lastfield->variable.name = (const char*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_field_description_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->lastfield->variable.description = (const char*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_field_details_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->format->details = strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_field_map_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (dss_map_end(tag, fp, disc))
		return -1;
	flat->format->map = (Cxmap_t*)fp->data;
	return 0;
}

static int
flat_field_con_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->constraint = (Cxconstraint_t*)fp->data;
	return 0;
}

static int
flat_field_type_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			s;
	Cxtype_t*		t;

	(void)cxattr(NiL, data, &s, flat->format, NiL);
	if (!*s)
		t = (Cxtype_t*)"number";
	else if (!(t = (Cxtype_t*)strdup(s)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	if (flat->format->flags & CX_FLOAT)
		flat->format->flags &= ~(CX_STRING|CX_BUFFER|CX_UNSIGNED|CX_INTEGER);
	else if (flat->format->flags & CX_UNSIGNED)
	{
		flat->format->flags &= ~(CX_STRING|CX_BUFFER);
		flat->format->flags |= CX_UNSIGNED|CX_INTEGER;
	}
	else if (!(flat->format->flags & (CX_STRING|CX_BUFFER|CX_INTEGER)))
	{
		if (streq(s, "string"))
			flat->format->flags |= CX_STRING;
		else if (streq(s, "buffer"))
			flat->format->flags |= CX_BUFFER;
	}
	if (flat->format == &flat->lastfield->variable.format)
		flat->lastfield->variable.type = t;
	else
		flat->lastfield->physical.type = t;
	return 0;
}

static int
flat_field_delimiter_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->delimiter = *data;
	if (flat->format->code != CC_NATIVE && (fp->attr & TAG_ATTR_conv))
		flat->format->delimiter = ccmapc(flat->format->delimiter, CC_NATIVE, flat->format->code);
	return 0;
}

static int
flat_field_multiple_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (strtol(data, NiL, 0) > 0)
		flat->format->flags |= CX_MULTIPLE;
	else
		flat->format->flags &= ~CX_MULTIPLE;
	return 0;
}

static int
flat_field_escape_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->escape = *data;
	return 0;
}

static int
flat_field_quotebegin_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->quotebegin = *data;
	return 0;
}

static int
flat_field_quoteend_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->quoteend = *data;
	return 0;
}

static int
flat_field_quoteall_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (strtol(data, NiL, 0) > 0)
		flat->format->flags |= CX_QUOTEALL;
	else
		flat->format->flags &= ~CX_QUOTEALL;
	return 0;
}

static int
flat_field_fixedpoint_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->format->fixedpoint = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_field_width_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	if (isdigit(*data))
	{
		flat->format->width = strtoul(data, &e, 0);
		if (*e)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
			return -1;
		}
	}
	else if (!(flat->lastfield->width = (Cxexpr_t*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_field_remainder_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->format->width = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	flat->format->flags |= CX_VARIABLE;
	return 0;
}

static int
flat_field_fill_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format->fill = *data;
	return 0;
}

static int
flat_field_base_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->format->base = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_field_codeset_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if ((flat->format->code = ccmapid(data)) < 0)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: unknown character code set", data);
		return -1;
	}
	return 0;
}

static int
flat_array_delimiter_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->array->delimiter = *data;
	return 0;
}

static int
flat_array_size_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	if (isdigit(*data))
	{
		flat->array->size = strtoul(data, &e, 0);
		if (*e)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
			return -1;
		}
	}
	else if (!(flat->array->variable = (Cxvariable_t*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static Tags_t	tags_array[] =
{
	"SIZE",		"Fixed array size or variable size field name.",
			0,0,flat_array_size_dat,0,
	"PHYSICAL",	"Physical (file representation) details; the"
			" remaining tags may appear inside <PHYSICAL>."
			" Outside of <PHYSICAL> the tags provide logical"
			" (print representation) details. Logical details"
			" are used as physical defaults.",
			0,flat_array_physical_beg,0,flat_array_physical_end,
	"DELIMITER",	"Array value delimiter character.",
			0,0,flat_array_delimiter_dat,0,
	0
};

static Tags_t*
flat_array_physical_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (name)
	{
		if (!flat->lastfield->physical.array && !(flat->lastfield->physical.array = newof(0, Cxarray_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		flat->array = flat->lastfield->physical.array;
	}
	return &tags_array[2];
}

static int
flat_array_physical_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->array = flat->lastfield->variable.array;
	return 0;
}

static Tags_t*
flat_array_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (name)
	{
		if (!flat->lastfield->variable.array && !(flat->lastfield->variable.array = newof(0, Cxarray_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		flat->array = flat->lastfield->variable.array;
	}
	return &tags_array[0];
}

static Key_t*
addkey(const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	register Key_t*		key;
	register Key_t*		old;
	register Dt_t*		dict;

	if (!(key = newof(0, Key_t, 1, strlen(data))))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	strcpy(key->name, data);
	key->field = flat->lastfield;
	if (streq(key->name, "*"))
		flat->lastfield->keyed = 2;
	else
	{
		flat->lastfield->keyed = 1;
		dict = ((Field_t*)flat->lastfield->structure.parent)->table->dict;
		if (old = (Key_t*)dtmatch(dict, key->name))
		{
			if (!old->qualification)
				error(1, "%s: %s: field key is ambiguous -- qualification required", old->field->variable.name, key->name);
			key->next = old;
			dtdelete(dict, old);
		}
		dtinsert(dict, key);
	}
	return key;
}

static int
flat_field_key_name_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Key_t*		key;

	if (fp->prev->data)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: key name already specified", data);
		return -1;
	}
	if (!(key = addkey(data, disc)))
		return -1;
	fp->prev->data = key;
	return 0;
}

static int
flat_field_key_qualification_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Key_t*		key;

	if (!(key = (Key_t*)fp->prev->data))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "key must be named");
		return -1;
	}
	if (!(key->qualification = strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
identifiers(register Table_t* tab, register const char* s, int flags, Tagdisc_t* disc)
{
	register int	c;
	register int	d;
	register char*	t;
	char*		e;
	regclass_t	f;
	char		m;

	while (c = *s++)
	{
		if (*s == '[')
		{
			switch (*(s + 1))
			{
			case ':':
				if (f = regclass(s + 1, &e))
				{
					s = (const char*)e;
					for (c = 0; c <= UCHAR_MAX; c++)
						if ((*f)(c))
							tab->id[c] |= flags;
					continue;
				}
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 2, "[%s: unknown character class", s);
				return -1;
			case '.':
			case '=':
				if (regcollate(s, &e, &m, 1, NiL) == 1)
				{
					s = (const char*)e;
					tab->id[m] |= flags;
					continue;
				}
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 2, "[%s: invalid collation element", s);
				return -1;
			}
		}
		tab->id[c] |= flags;
		if (*s == '-' && (d = *(s + 1)))
		{
			s += 2;
			if (c == 'A' && d == 'Z')
				t = "BCDEFGHIJKLMNOPQRSTUVWXYZ";
			else if (c == 'a' && d == 'z')
				t = "bcdefghijklmnopqrstuvwxyz";
			else if (c == '0' && d == '9')
				t = "123456789";
			else
			{
				while (++c <= d)
					tab->id[c] |= flags;
				continue;
			}
			while (c = *t++)
				tab->id[c] |= flags;
		}
	}
	return 0;
}

static Table_t*
table(Tagframe_t* fp, Tagdisc_t* disc)
{
	Field_t*	p;

	if (!(p = fp->data) && !(p = (Field_t*)fp->prev->data))
		p = &((Flat_t*)disc)->root;
	if (!p->table)
		p->table = tabinit((Flat_t*)disc, (Dssdisc_t*)disc);
	return p->table;
}

static int
flat_field_key_id1_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	return identifiers(table(fp, disc), data, 3, disc);
}

static int
flat_field_key_id2_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	return identifiers(table(fp, disc), data, 2, disc);
}

static int
flat_field_key_span_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	table(fp, disc)->span = *data ? *data : -1;
	return 0;
}

static int
flat_field_key_unknown_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	table(fp, disc)->unknown = strtol(data, NiL, 0) == 1;
	return 0;
}

static Tags_t	tags_key[] =
{
	"NAME",		"Key name.",
			0,0,flat_field_key_name_dat,0,
	"QUALIFICATION","Key qualification expression; the key is active"
			" for the current record when the expression evaluates"
			" non-zero. Used to disambiguate conflicting key"
			" names. Why design data with conflicts?",
			0,0,flat_field_key_qualification_dat,0,
	"ID1",		"Characters that may appear anywhere in key names,"
			" interpreted as the contents of an RE character"
			" class. The default is \b[:alpha:]_\b.",
			0,0,flat_field_key_id1_dat,0,
	"ID2",		"Characters that may appear after the first character"
			" in key names, interpreted as the contents of an RE"
			" character class. The default is"
			" \b[:alnum:]_.,-\b.",
			0,0,flat_field_key_id2_dat,0,
	"SPAN",		"The value is the key assignment character. If set"
			" then key values may span the field separator"
			" character up to the end of record or the next"
			" \akey\a\aspan\a\avalue\a. Such data is fraught with"
			" ambiguities. One must suppose that it never occurred"
			" to the writers that the data would someday be read.",
			0,0,flat_field_key_span_dat,0,
	"UNKNOWN",	"Span unknown but otherwise syntactically correct"
			" keys.",
			0,0,flat_field_key_unknown_dat,0,
	0
};

static Tags_t*
flat_field_key_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (name && !((Field_t*)flat->lastfield->structure.parent)->table && !(((Field_t*)flat->lastfield->structure.parent)->table = tabinit(flat, (Dssdisc_t*)disc)))
		return 0;
	return &tags_key[0];
}

static int
flat_field_key_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	return addkey(data, disc) ? 0 : -1;
}

static Tags_t	tags_flat_field[] =
{
	"NAME",		"Field name.",
			0,0,flat_field_name_dat,0,
	"DESCRIPTION",	"Field description.",
			0,0,flat_field_description_dat,0,
	"PHYSICAL",	"Physical (file representation) details; the"
			" remaining tags may appear inside <PHYSICAL>."
			" Outside of <PHYSICAL> the tags provide logical"
			" (print representation) details. Logical details"
			" are used as physical defaults.",
			0,flat_field_physical_beg,0,flat_field_physical_end,
	"MAP",		"Field value map;"
			" either a map reference name or a map definition.",
			0,dss_map_beg,dss_map_dat,flat_field_map_end,
	"TYPE",		"Field type. The intrinsic types are number and"
			" string. Other types are defined in optional"
			" method and schema libraries.",
			0,0,flat_field_type_dat,0,
	"DETAILS",	"An optional type-specific comma-separated string"
			" of name=value pairs.",
			0,0,flat_field_details_dat,0,
	"DELIMITER",	"Field delimiter character.",
			0,0,flat_field_delimiter_dat,0,
	"MULTIPLE",	"Multiple adjacent delimiters are equivalent to one.",
			0,0,flat_field_multiple_dat,0,
	"ESCAPE",	"Field delimiter and/or quote escape character.",
			0,0,flat_field_escape_dat,0,
	"QUOTE",	"Field quote begin and end character.",
			0,0,flat_field_quotebegin_dat,0,
	"QUOTEBEGIN",	"Field quote begin character.",
			0,0,flat_field_quotebegin_dat,0,
	"QUOTEEND",	"Field quote end character.",
			0,0,flat_field_quoteend_dat,0,
	"QUOTEALL",	"Field quotes are the first and last characetrs.",
			0,0,flat_field_quoteall_dat,0,
	"CODESET",	"Field codeset name; one of { native ascii ebcdic }.",
			0,0,flat_field_codeset_dat,0,
	"FIXEDPOINT",	"Fixed point width.",
			0,0,flat_field_fixedpoint_dat,0,
	"WIDTH",	"Field fixed width in bytes.",
			0,0,flat_field_width_dat,0,
	"REMAINDER",	"Field is variable length up to the end of record.",
			0,0,flat_field_remainder_dat,0,
	"BASE",		"Numeric field representation base.",
			0,0,flat_field_base_dat,0,
	"FILL",		"Fixed width field fill character.",
			0,0,flat_field_fill_dat,0,
	"KEY",		"name=value keyed field details. Keyed fields are"
			" optional and may appear in any order after the"
			" positional fields. All unknown keys and invalid"
			" key data are passed to the key named \b*\b; if there"
			" is no \b*\b key or if \b--verbose\b is on then an"
			" error message is emitted for each occurrence. If"
			" \b<SPAN>\b and \b<UNKNOWN>\b are enabled then"
			" unknown keys are spanned and unknown/invalid"
			" key messages are disabled.",
			0,flat_field_key_beg,flat_field_key_dat,0,
	"CONSTRAINT",	"Field value constraints. Constraints, when"
			" enabled, are applied to each record as it is read.",
			0,dss_con_beg,dss_con_dat,flat_field_con_end,
	"ARRAY",	"Array info.",
			0,flat_array_beg,0,0,
	"FIELD",	"Field structure.",
			0,flat_field_beg,0,flat_field_end,
	0
};

static Tags_t*
flat_field_physical_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (name)
		flat->format = &flat->lastfield->physical.format;
	return &tags_flat_field[3];
}

static int
flat_field_physical_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->format = &flat->lastfield->variable.format;
	return 0;
}

static Tags_t*
flat_field_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	Field_t*		f;
	Field_t*		p;
	Field_t*		t;

	if (name)
	{
		if (!(f = newof(0, Field_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		f->variable.structure = &f->structure;
		if (p = fp->data)
		{
			f->structure.parent = p->structure.parent;
			f->structure.level = p->structure.level;
			((Field_t*)fp->tail)->structure.next = &f->variable;
		}
		else
		{
			fp->data = f;
			if (p = (Field_t*)fp->prev->data)
			{
				f->structure.parent = &p->variable;
				f->structure.level = p->structure.level + 1;
				if (t = (Field_t*)fp->prev->head)
					t->structure.next = &f->variable;
				else
					p->structure.members = &f->variable;
				fp->prev->head = f;
			}
			else
			{
				f->structure.level = 1;
				f->structure.parent = &flat->root.variable;
			}
		}
		fp->tail = f;
		if (!flat->lastfield)
			flat->fields = f;
		else
			flat->lastfield->next = f;
		flat->lastfield = f;
		flat->format = &f->variable.format;
		f->variable.format.delimiter = f->physical.format.delimiter = flat->delimiter;
		f->variable.format.flags = f->physical.format.flags = flat->flags;
		f->variable.format.escape = f->physical.format.escape = flat->escape;
		f->variable.format.quotebegin = f->physical.format.quotebegin = flat->quotebegin;
		f->variable.format.quoteend = f->physical.format.quoteend = flat->quoteend;
		f->variable.format.code = f->physical.format.code = CC_NATIVE;
		f->variable.index = flat->nfields++;
		f->variable.type = (Cxtype_t*)"void";
	}
	return &tags_flat_field[0];
}

static int
flat_field_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (flat->lastfield && !flat->lastfield->variable.name)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "field name must be specified");
		return -1;
	}
	return 0;
}

static int
flat_size_type_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	switch (((Size_t*)fp->prev->data)->type = *data)
	{
	case 'a':
	case 'b':
	case 'd':
	case 'e':
	case 'l':
	case 'm':
		break;
	default:
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: invalid size field type", data);
		return -1;
	}
	return 0;
}

static int
flat_size_width_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	char*	e;

	((Size_t*)fp->prev->data)->width = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_size_fixed_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	char*	e;

	((Size_t*)fp->prev->data)->fixed = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_size_offset_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	char*	e;

	((Size_t*)fp->prev->data)->offset = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_size_base_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	char*	e;

	((Size_t*)fp->prev->data)->base = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static Tags_t	tags_flat_size[] =
{
	"TYPE",		"Size field physical type. The types are:"
			" ascii - ascii text digits,"
			" be_t - big endian binary,"
			" ebcdic - ebcdic text digits,"
			" le_t - little endian binary,"
			" magic - binary with same byte order as the"
			" header magic number.",
			0,0,flat_size_type_dat,0,
	"WIDTH",	"Size field width in bytes.",
			0,0,flat_size_width_dat,0,
	"OFFSET",	"Size field offset in bytes.",
			0,0,flat_size_offset_dat,0,
	"BASE",		"Size field field representation base.",
			0,0,flat_size_base_dat,0,
	"SIZE",		"Size field total width in bytes, <WIDTH>+<OFFSET>"
			" by default. A + prefix specifies that the size"
			" field total width must be added to the computed"
			" size to determine the record length.",
			0,0,flat_size_width_dat,0,
	"FIXED",	"Fixed record size.",
			0,0,flat_size_fixed_dat,0,
	0
};

static Tags_t*
flat_size_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	Size_t*			z;

	if (name)
	{
		if (!(z = newof(0, Size_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		if (*name == 'B')
			flat->block = z;
		else
			flat->record = z;
		fp->data = z;
	}
	return &tags_flat_size[0];
}

static int
flat_size_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	register Size_t*	z = (Size_t*)fp->data;

	if (z->size < (z->offset + z->width))
		z->size = z->offset + z->width;
	z->reserve = z->fixed;
	if (z->size)
	{
		z->fixed = 0;
		if (z->reserve < z->size)
			z->reserve = z->size;
	}
	if (z->type == 'e')
	{
		if (!(z->buf = newof(0, char, z->reserve, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return -1;
		}
		if (!flat->e2a)
			flat->e2a = ccmap(CC_EBCDIC_O, CC_ASCII);
	}
	return 0;
}

static int
flat_section_count_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->section->count = strtoul(data, &e, 0);
	if (*e && (*e != '*' || *(e + 1)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_section_delimiter_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->section->delimiter = *data;
	return 0;
}

static int
flat_section_pattern_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	int			code;

	if (!(flat->section->re = newof(0, regex_t, 1, 0)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	if (code = regcomp(flat->section->re, data, REG_AUGMENTED|REG_LENIENT))
	{
		if (disc->errorf)
		{
			char	buf[256];

			regerror(code, flat->section->re, buf, sizeof(buf));
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "regular expression: %s: %s", data, buf);
		}
		return -1;
	}
	return 0;
}

static int
flat_section_size_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->section->size = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static Tags_t	tags_flat_section[] =
{
	"COUNT",	"Number of delimiters, patterns or sized blocks.",
			0,0,flat_section_count_dat,0,
	"DELIMITER",	"Delimiter character.",
			0,0,flat_section_delimiter_dat,0,
	"PATTERN",	"Regular expression; ignored if"
			" delimiter or size specified.",
			0,0,flat_section_pattern_dat,0,
	"SIZE",		"Fixed size in bytes.",
			0,0,flat_section_size_dat,0,
	0
};

static Tags_t*
flat_header_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	Section_t*		s;

	if (name)
	{
		if (!(s = newof(0, Section_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		s->count = 1;
		s->delimiter = -1;
		if (!flat->lastheader)
			flat->header = flat->lastheader = s;
		else
			flat->lastheader = flat->lastheader->next = s;
		flat->section = s;
	}
	return &tags_flat_section[0];
}

static Tags_t*
flat_trailer_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	Section_t*		s;

	if (name)
	{
		if (!(s = newof(0, Section_t, 1, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
			return 0;
		}
		s->count = 1;
		s->delimiter = -1;
		if (!flat->lasttrailer)
			flat->trailer = flat->lasttrailer = s;
		else
			flat->lasttrailer = flat->lasttrailer->next = s;
		flat->section = s;
	}
	return &tags_flat_section[0];
}

static int
flat_name_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->meth.name = strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_print_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->meth.print = strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_description_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->meth.description = (const char*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_library_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	register Library_t*	p;

	if (!(p = newof(0, Library_t, 1, strlen(data))))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	strcpy(p->name, data);
	if (!flat->lastlibrary)
		flat->libraries = flat->lastlibrary = p;
	else
		flat->lastlibrary = flat->lastlibrary->next = p;
	return 0;
}

static int
flat_magic_string_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->magic->string = (const char*)strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	if (!flat->magic->length)
		flat->magic->length = strlen(flat->magic->string);
	return 0;
}

static int
flat_magic_number_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->magic->number = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static int
flat_magic_size_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->magic->size = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static Value_t	flat_swap_val[] =
{
	"none",		SWAP_none,
	"native",	SWAP_native,
	"be",		SWAP_be,
	"le",		SWAP_le,
};

static int
flat_magic_swap_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	register Value_t*	v;
	char*			e;

	for (v = flat_swap_val; v < &flat_swap_val[elementsof(flat_swap_val)]; v++)
		if (!strcasecmp(data, v->name))
		{
			flat->magic->swap = v->value;
			break;
		}
	if (v >= &flat_swap_val[elementsof(flat_swap_val)])
	{
		flat->magic->swap = strtoul(data, &e, 0);
		if (*e)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
			return -1;
		}
	}
	if (flat->magic->swap >= 0)
		flat->magic->swap = SWAP(flat->magic->swap);
	return 0;
}

static int
flat_magic_version_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	char*			e;

	flat->magic->version = strtoul(data, &e, 0);
	if (*e)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
		return -1;
	}
	return 0;
}

static Tags_t	tags_flat_magic[] =
{
	"STRING",	"Magic string value.",
			0,0,flat_magic_string_dat,0,
	"NUMBER",	"Magic number value.",
			0,0,flat_magic_number_dat,0,
	"LENGTH",	"Magic number/string size in bytes.",
			0,0,flat_magic_size_dat,0,
	"SIZE",		"Magic header size in bytes. If omitted then an AST"
			"magic header is assumed.",
			0,0,flat_magic_size_dat,0,
	"SWAP",		"Magic header binary field swap operation:"
			" native - native byte order (default),"
			" be - big endian swap,"
			" le - little endian swap,"
			" 1|2|4 - swap OR of 1:bytes 2:shorts 4:ints.",
			0,0,flat_magic_swap_dat,0,
	"VERSION",	"Magic version stamp, either YYYYMMDD or 0xWWXXYYZZ.",
			0,0,flat_magic_version_dat,0,
	0
};

static Tags_t*
flat_magic_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->magic = newof(0, Magic_t, 1, 0)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	flat->magic->swap = SWAP_native;
	return &tags_flat_magic[0];
}

static int
flat_magic_end(Tag_t* tag, Tagframe_t* fp, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (flat->magic)
	{
		if (!flat->magic->length)
			flat->magic->length = 4;
		if (!flat->magic->size)
			flat->magic->size = flat->magic->length;
		if (flat->magic->size < flat->magic->length)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "magic header size %u is smaller than string/number length %u", flat->magic->size, flat->magic->length);
			return -1;
		}
		if (!flat->magic->string && (flat->magic->length > 4 || (flat->magic->length & (flat->magic->length - 1))))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "magic number length %u must be a power of 2 less than or equal to 4", flat->magic->length);
			return -1;
		}
	}
	return 0;
}

static int
flat_physical_swap_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;
	register Value_t*	v;
	char*			e;

	for (v = flat_swap_val; v < &flat_swap_val[elementsof(flat_swap_val)]; v++)
		if (!strcasecmp(data, v->name))
		{
			flat->swap = v->value;
			break;
		}
	if (v >= &flat_swap_val[elementsof(flat_swap_val)])
	{
		flat->swap = strtoul(data, &e, 0);
		if (*e)
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: invalid number", data);
			return -1;
		}
	}
	if (flat->swap >= 0)
		flat->swap = SWAP(flat->swap);
	return 0;
}

static int
flat_compress_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (!(flat->meth.compress = strdup(data)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	return 0;
}

static int
flat_physical_continue_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->continuator = *data;
	return 0;
}

static int
flat_physical_delimiter_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->delimiter = *data;
	return 0;
}

static int
flat_physical_escape_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->escape = *data;
	return 0;
}

static int
flat_physical_quotebegin_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->quotebegin = *data;
	return 0;
}

static int
flat_physical_quoteend_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->quoteend = *data;
	return 0;
}

static int
flat_physical_quoteall_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (strtol(data, NiL, 0) > 0)
		flat->flags |= CX_QUOTEALL;
	else
		flat->flags &= ~CX_QUOTEALL;
	return 0;
}

static int
flat_physical_multiple_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if (strtol(data, NiL, 0) > 0)
		flat->flags |= CX_MULTIPLE;
	else
		flat->flags &= ~CX_MULTIPLE;
	return 0;
}

static int
flat_physical_codeset_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	if ((flat->code = ccmapid(data)) < 0)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "%s: unknown character code set", data);
		return -1;
	}
	return 0;
}

static int
flat_physical_terminator_dat(Tag_t* tag, Tagframe_t* fp, const char* data, Tagdisc_t* disc)
{
	register Flat_t*	flat = (Flat_t*)disc;

	flat->terminator = *data;
	if (flat->code != CC_NATIVE && (fp->attr & TAG_ATTR_conv))
		flat->terminator = ccmapc(flat->terminator, CC_NATIVE, flat->code);
	return 0;
}

static Tags_t	tags_flat_physical_key[] =
{
	"ID1",		"Characters that may appear anywhere in key names,"
			" interpreted as the contents of an RE character"
			" class. The default is \b[:alpha:]_\b.",
			0,0,flat_field_key_id1_dat,0,
	"ID2",		"Characters that may appear after the first character"
			" in key names, interpreted as the contents of an RE"
			" character class. The default is"
			" \b[:alnum:]_.,-\b.",
			0,0,flat_field_key_id2_dat,0,
	"SPAN",		"The value is the key assignment character. If set"
			" then key values may span the field separator"
			" character up to the end of record or the next"
			" \akey\a\aspan\a\avalue\a. Such data is frought with"
			" ambiguities. One must suppose that it never occurred"
			" to the writers that the data would someday be read.",
			0,0,flat_field_key_span_dat,0,
	"UNKNOWN",	"Span unknown but otherwise syntactically correct"
			" keys.",
			0,0,flat_field_key_unknown_dat,0,
	0
};

static Tags_t*
flat_physical_key_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	return &tags_flat_physical_key[0];
}

static Tags_t	tags_flat_physical[] =
{
	"SWAP",		"Binary record field swap operation:"
			" none - don't swap (default),"
			" native - swap to match the magic header NUMBER,"
			" be - big endian swap,"
			" le - little endian swap,"
			" 1|2|4 - swap OR of 1:bytes 2:shorts 4:ints.",
			0,0,flat_physical_swap_dat,0,
	"DELIMITER",	"Default field delimiter and continuation replacement"
			" character.",
			0,0,flat_physical_delimiter_dat,0,
	"ESCAPE",	"Default field delimiter and/or quote escape character.",
			0,0,flat_physical_escape_dat,0,
	"QUOTE",	"Default field quote begin and end character.",
			0,0,flat_physical_quotebegin_dat,0,
	"QUOTEBEGIN",	"Default field quote begin character.",
			0,0,flat_physical_quotebegin_dat,0,
	"QUOTEEND",	"Default field quote end character.",
			0,0,flat_physical_quoteend_dat,0,
	"QUOTEALL",	"Field quotes are the first and last characetrs.",
			0,0,flat_physical_quoteall_dat,0,
	"MULTIPLE",	"Multiple adjacent delimiters are equivalent to one.",
			0,0,flat_physical_multiple_dat,0,
	"CODESET",	"Default records codeset name; one of { native ascii ebcdic }.",
			0,0,flat_physical_codeset_dat,0,
	"CONTINUE",	"Terminator continuation (escape) character.",
			0,0,flat_physical_continue_dat,0,
	"TERMINATOR",	"Default record termination character.",
			0,0,flat_physical_terminator_dat,0,
	"KEY",		"Default field key attributes.",
			0,flat_physical_key_beg,0,0,
};

static Tags_t*
flat_physical_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	return &tags_flat_physical[0];
}

static Tags_t	tags_flat[] =
{
	"NAME",		"Schema name.",
			0,0,flat_name_dat,0,
	"DESCRIPTION",	"Schema description.",
			0,0,flat_description_dat,0,
	"IDENT",	"Schema ident string.",
			0,0,0,0,
	"LIBRARY",	"Required type/map library name;"
			" more than one library may be specified.",
			0,0,flat_library_dat,0,
	"MAGIC",	"File magic number (identification header)"
			" section definitions.",
			0,flat_magic_beg,0,flat_magic_end,
	"PRINT",	"Default {print} query format.",
			0,0,flat_print_dat,0,
	"COMPRESS",	"Preferred compression method; compression is applied"
			" by the {compress} query.",
			0,0,flat_compress_dat,0,
	"HEADER",	"File header section definitions.",
			0,flat_header_beg,0,0,
	"TRAILER",	"File trailer section definitions.",
			0,flat_trailer_beg,0,0,
	"BLOCK",	"Block size definitions.",
			0,flat_size_beg,0,flat_size_end,
	"RECORD",	"Record size definitions; same as <BLOCK>.",
			0,flat_size_beg,0,flat_size_end,
	"PHYSICAL",	"Default physical record and field attributes.",
			0,flat_physical_beg,0,0,
	"FIELD",	"Schema field list.",
			0,flat_field_beg,0,flat_field_end,
	0
};

static Tags_t*
flat_beg(Tag_t* tag, Tagframe_t* fp, const char* name, Tagdisc_t* disc)
{
	return &tags_flat[0];
}

static Tags_t	tags[] =
{
	"FLAT",		"Flat method schema.",
			0,flat_beg,0,0,
	"MAP",		"Field value map;"
			" either a map reference name or a map definition.",
			0,dss_map_beg,dss_map_dat,dss_map_end,
	"METHOD",	"Method name; must be flat.",
			0,0,0,0,
	0
};

/*
 * outal rententive support
 */

static int
tabs(Sfio_t* op, int cur, int nxt)
{
	cur = (nxt - cur - 1) / 8 + 1;
	while (cur-- > 0)
		sfputc(op, '\t');
	return nxt;
}

/*
 * fill in type and format defaults
 */

static void
defaults(register Cxtype_t* type, register Cxformat_t* format, int binary, Dssdisc_t* disc)
{
	register char*	s;
	Cxstate_t*	state = cxstate(disc);
	Cxtype_t*	base;
	char		details[16];

	if (!(format->flags & (CX_INTEGER|CX_FLOAT|CX_STRING|CX_BUFFER)))
		format->flags |= type->format.flags & (CX_BINARY|CX_UNSIGNED|CX_INTEGER|CX_FLOAT|CX_STRING|CX_BUFFER);
	else if ((type->format.flags & CX_BINARY) && (format->flags & (CX_INTEGER|CX_FLOAT)))
		format->flags |= CX_BINARY;
	if (!type->internalf || !type->externalf)
	{
		if (cxisvoid(type))
			base = state->type_void;
		else if (cxisstring(type))
			base = state->type_string;
		else if (cxisbuffer(type))
			base = state->type_buffer;
		else
		{
			base = state->type_number;
			if (!(format->flags & CX_FLOAT))
			{
				s = details;
				*s++ = '%';
				if (format->fill > 0)
					*s++ = format->fill;
				if (format->width)
					s += sfsprintf(s, &details[sizeof(details)] - s, "%d", format->width);
				*s++ = 'l';
				*s++ = 'l';
				switch (format->base)
				{
				default:
					if (format->base < 64)
					{
						*s++ = '.';
						*s++ = '.';
						s += sfsprintf(s, &details[sizeof(details)] - s, "%d", format->base);
						break;
					}
					/*FALLTHROUGH*/
				case 0:
				case 10:
					*s++ = (format->flags & CX_UNSIGNED) ? 'u' : 'd';
					break;
				case 8:
					*s++ = 'o';
					break;
				case 16:
					*s++ = 'x';
					break;
				}
				*s = 0;
				format->details = strdup(details);
			}
		}
		if (!type->internalf)
			type->internalf = base->internalf;
		if (!type->externalf)
			type->externalf = base->externalf;
	}
}

/*
 * methf
 */

extern Dsslib_t	dss_lib_flat;

static Dssmeth_t*
flatmeth(const char* name, const char* options, const char* schema, Dssdisc_t* disc, Dssmeth_t* meth)
{
	register Flat_t*	flat;
	register Library_t*	p;
	register Field_t*	f;
	Field_t*		g;
	Tag_t*			tag;
	Sfio_t*			sp;
	char*			s;
	ssize_t			n;
	int			i;
	int			errors;
	int			fixed;
	char			path[PATH_MAX];

	if (!(flat = newof(0, Flat_t, 1, 0)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	flat->basemeth = meth;
	flat->meth = *meth;
	flat->meth.data = flat;
	taginit(&flat->dsstagdisc.tagdisc, disc->errorf);
	flat->dsstagdisc.tagdisc.id = DSS_ID;
	flat->dsstagdisc.disc = disc;
	flat->dsstagdisc.meth = meth;
	flat->binary = 1;
	flat->code = CC_NATIVE;
	flat->swap = SWAP_none;
	flat->delimiter = flat->escape = flat->quotebegin = flat->quoteend = flat->terminator = flat->continuator = -1;
	sp = 0;
	if (options)
	{
		if (!(sp = sfstropen()))
			goto drop;
		sfprintf(sp, "%s", usage);
		if (tagusage(tags, sp, &flat->dsstagdisc.tagdisc))
			goto drop;
		sfprintf(sp, "}\n");
		if (dssoptlib(meth->cx->buf, &dss_lib_flat, sfstruse(sp), disc))
			goto drop;
		sfclose(sp);
		sp = 0;
		s = sfstruse(meth->cx->buf);
		for (;;)
		{
			switch (optstr(options, s))
			{
			case 'b':
				flat->binary = opt_info.num;
				continue;
			case 'e':
				flat->emptyspace = 1;
				continue;
			case 'h':
			case 'o':
			case 's':
				flat->list = opt_info.option[1];
				continue;
			case 'p':
				flat->prototype = 1;
				continue;
			case 'T':
				flat->test = opt_info.num;
				continue;
			case '?':
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, ERROR_USAGE|4, "%s", opt_info.arg);
				goto drop;
			case ':':
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 2, "%s", opt_info.arg);
				goto drop;
			}
			break;
		}
	}
	if (schema && *schema)
	{
		if (!(sp = dssfind(schema, NiL, DSS_VERBOSE, path, sizeof(path), disc)))
			return 0;
		if (!(tag = tagopen(sp, path, 1, &tags[0], &flat->dsstagdisc.tagdisc)) || tagclose(tag))
			goto drop;
		sfclose(sp);
		sp = 0;
		if (!flat->fields)
			goto invalid;
	}
	dtinsert(flat->meth.formats, &flat_format);
	for (p = flat->libraries; p; p = p->next)
		if (!dssload(p->name, disc))
			return 0;
	for (i = 0; i < elementsof(local_callouts); i++)
		if (cxaddcallout(flat->meth.cx, &local_callouts[i], disc))
			return 0;
	for (f = flat->fields; f; f = f->next)
	{
		if (cxaddvariable(flat->meth.cx, &f->variable, disc))
			return 0;
		if ((s = (char*)f->width) && !(f->width = keycomp(flat, s, disc)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: %s: invalid field width index", f->variable.name, s);
			return 0;
		}
		if (f->variable.array && (s = (char*)f->variable.array->variable) && !(f->variable.array->variable = (Cxvariable_t*)dtmatch(flat->meth.cx->variables, s)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: %s: unknown array index", f->variable.name, s);
			return 0;
		}
		defaults(f->variable.type, &f->variable.format, 0, disc);
		if (!(s = (char*)f->physical.type))
			f->physical.type = f->variable.type;
		else if (!(f->physical.type = cxtype(flat->meth.cx, s, disc)))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: %s: unknown type", f->variable.name, s);
			return 0;
		}
		defaults(f->physical.type, &f->physical.format, flat->binary, disc);
		if (!f->physical.format.delimiter)
			f->physical.format.delimiter = f->variable.format.delimiter;
		if (!(f->physical.format.flags & (CX_INTEGER|CX_FLOAT|CX_STRING|CX_BUFFER)))
			f->physical.format.flags |= f->variable.format.flags & (CX_BINARY|CX_UNSIGNED|CX_INTEGER|CX_FLOAT|CX_STRING|CX_BUFFER|CX_NUL);
		if (f->physical.format.flags & CX_BINARY)
		{
			if (!f->variable.format.width && f->physical.type)
				f->variable.format.width = f->physical.type->format.width;
		}
		else if (!f->physical.format.map)
			f->physical.format.map = f->variable.format.map;
		if (f->physical.format.flags & CX_BUFFER)
			f->variable.format.flags |= CX_BUFFER;
	}
	if (flat->lastfield && flat->terminator < 0)
		flat->terminator = flat->lastfield->physical.format.delimiter;
	if (flat->terminator < 0)
		flat->continuator = -1;
	else if (flat->continuator >= 0 && !(flat->buf = sfstropen()))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return 0;
	}
	errors = 0;
	fixed = 1;
	for (f = flat->fields; f; f = f->next)
	{
		if (!(f->physical.format.flags & (CX_BINARY|CX_STRING|CX_BUFFER)))
			flat->binary = 0;
		if (f->physical.format.delimiter < 0 && !f->physical.format.width && !f->width && !f->keyed)
		{
			if (f->structure.members)
			{
				f->structref = 1;
				continue;
			}
			errors++;
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: a field delimiter or size must be specified", f->variable.name);
		}
		else if (f->structure.members)
		{
#if 0
			if (f->physical.format.width || f->width)
				/*OK*/;
			else if (disc->errorf)
				(*disc->errorf)(NiL, disc, 1, "%s: structure attributes ignored", f->variable.name);
#endif
			continue;
		}
		if (!f->variable.type)
		{
			f->variable.type = (Cxtype_t*)"string";
			f->variable.format.flags |= CX_STRING;
		}
		if (f->physical.format.delimiter == flat->terminator && f != flat->lastfield)
			flat->terminator = -1;
		if (f->physical.format.width)
		{
			flat->fixed += f->physical.format.width;
			if (f->width)
				flat->variable = SF_LOCKR;
		}
		else
			fixed = 0;
		if (f->physical.format.quotebegin >= 0 && f->physical.format.quoteend < 0)
			f->physical.format.quoteend = f->physical.format.quotebegin;
		if (f->physical.format.quotebegin < 0)
			f->physical.format.flags &= ~CX_QUOTEALL;
		f->map = ccmap(f->physical.format.code, CC_NATIVE);
	}
	if (errors)
		return 0;
	if (flat->block)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "blocked data not supported yet");
		return 0;
	}
	if (flat->record)
	{
		if (flat->record->fixed && flat->fixed > flat->record->reserve)
		{
			flat->truncate.fixed = 1;
			n = flat->record->reserve;
			for (f = flat->fields; f; f = f->next)
			{
				if (f->width && f->physical.format.width)
				{
					g = f;
					while (g = g->next)
						n -= g->physical.format.width;
					if (n > 0)
					{
						if (disc->errorf)
							(*disc->errorf)(NiL, disc, 1, "%s: maximum variable field size shortened from %d to %d to comply with fixed record size %d", f->variable.name, f->physical.format.width, n, flat->record->reserve);
						f->physical.format.width = n;
						flat->truncate.fixed = 0;
					}
					break;
				}
				n -= f->physical.format.width;
			}
		}
		flat->fixed = flat->record->reserve;
	}
	else if (!fixed)
	{
		flat->fixed = 0;
		flat->binary = 0;
	}
	return &flat->meth;
 invalid:
	if (disc->errorf)
		(*disc->errorf)(NiL, disc, 2, "%s: invalid schema", options);
 drop:
	free(flat);
	if (sp)
		sfclose(sp);
	return 0;
}

/*
 * openf
 */

static int
flatopen(Dss_t* dss, Dssdisc_t* disc)
{
	Flat_t*			flat = (Flat_t*)dss->meth->data;
	register Field_t*	f;
	register Cxvariable_t*	v;
	char*			a;
	char*			s;
	char*			p;
	char*			t;
	char*			u;
	int			offset;
	int			fixed;
	int			pad;
	int			m;
	int			n;
	char			tmp[2];

	if (flat)
	{
		if (flat->prototype)
		{
			for (f = flat->fields; f; f = f->next)
			{
				if ((n = f->variable.format.delimiter) < 0 && (n = f->physical.format.delimiter) < 0)
					n = f->next ? '|' : '\n';
				sfprintf(sfstdout, "%s%c", cxisvoid(f->variable.type) ? "" : f->variable.name, n);
			}
		}
		switch (flat->list)
		{
		case 'h':
			if (!(a = newof(0, char, 3 * (strlen(flat->meth.name) + 1), 0)))
			{
				if (disc->errorf)
					(*disc->errorf)(dss, disc, ERROR_SYSTEM|2, "out of space");
				return -1;
			}
			p = a;
			for (s = (char*)flat->meth.name; *s; s++)
				*a++ = isalnum(*s) ? *s : '_';
			*a++ = 0;
			if (!isalpha(*p))
				*p = '_';
			t = a;
			a = strcopy(a, flat->meth.name);
			*a++ = 0;
			if (islower(*t))
				*t = toupper(*t);
			u = a;
			for (s = p; *s; s++)
				*a++ = islower(*s) ? toupper(*s) : *s;
			*a = 0;
			sfprintf(sfstdout, "/*\n");
			sfprintf(sfstdout, " * %s dynamic interface\n", flat->meth.name);
			if (flat->meth.description)
				sfprintf(sfstdout, " * %s\n", flat->meth.description);
			sfprintf(sfstdout, " */\n\n");
			sfprintf(sfstdout, "#define %s_RECORD(data)	(_%s_record_=(%s_record_t*)DSSDATA(data))\n", u, p, t);
			sfprintf(sfstdout, "\n");
			sfprintf(sfstdout, "typedef Cxvalue_t* (*%s_get_f)(void*,int);\n", t);
			sfprintf(sfstdout, "\n");
			sfprintf(sfstdout, "typedef struct %s_field_s		/* record field			*/\n", t);
			sfprintf(sfstdout, "{\n");
			sfprintf(sfstdout, "	Cxvalue_t	value;		/* value (first for dynamic Q)	*/\n");
			sfprintf(sfstdout, "	void*		field;		/* static field info		*/\n");
			sfprintf(sfstdout, "	size_t		off;		/* record data offset		*/\n");
			sfprintf(sfstdout, "	size_t		siz;		/* record data size		*/\n");
			sfprintf(sfstdout, "	unsigned int	serial;		/* read serial number		*/\n");
			sfprintf(sfstdout, "	unsigned int	keyed;		/* keyed serial number		*/\n");
			sfprintf(sfstdout, "} %s_field_t;\n", t);
			sfprintf(sfstdout, "\n");
			sfprintf(sfstdout, "typedef struct %s_record_s		/* current record info		*/\n", t);
			sfprintf(sfstdout, "{\n");
			sfprintf(sfstdout, "	%s_field_t*	fields;		/* fields (first for dynamic Q)	*/\n", t);
			sfprintf(sfstdout, "	%s_get_f	getf;		/* getf (second for dynamic Q)	*/\n", t);
			sfprintf(sfstdout, "} %s_record_t;\n", t);
			sfprintf(sfstdout, "\n");
			sfprintf(sfstdout, "static %s_record_t*	_%s_record_;\n", t, p);
			sfprintf(sfstdout, "\n");
			for (f = flat->fields; f; f = f->next)
			{
				if (f->structure.members)
					continue;
				n = sfprintf(sfstdout, "#define %s_%s", p, f->variable.name);
				n = tabs(sfstdout, n, 32);
				n += sfprintf(sfstdout, "((*_%s_record_->getf)(_%s_record_,%d)->", p, p, f->variable.index);
				if (f->variable.format.flags & (CX_BUFFER|CX_STRING))
				{
					a = (f->variable.format.flags & CX_STRING) ? "string" : "buffer";
					n += sfprintf(sfstdout, "%s.data)", a);
				}
				else
					n += sfprintf(sfstdout, "number)");
				offset += m * f->physical.format.width;
				if (f->variable.description)
				{
					tabs(sfstdout, n, 64);
					sfprintf(sfstdout, "/* %s */\n", f->variable.description);
				}
				else
					sfputc(sfstdout, '\n');
				if (f->variable.format.flags & (CX_BUFFER|CX_STRING))
				{
					n = sfprintf(sfstdout, "#define %s_%s_size", p, f->variable.name);
					tabs(sfstdout, n, 32);
					sfprintf(sfstdout, "((*_%s_record_->getf)(_%s_record_,%d)->%s.size)\n", p, p, f->variable.index, a);
				}
			}
			break;
		case 'o':
			if (!flat->fixed)
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 2, "variable width record offsets not supported");
				return -1;
			}
			/* HERE: under construction */
			offset = 0;
			pad = 0;
			for (f = flat->fields; f; f = f->next)
			{
				if (f->structure.members)
				{
					f->structure.size = offset;
					continue;
				}
				a = p = s = "";
				m = 1;
				n = 0;
				if (f->variable.array)
				{
					if (f->variable.array->size)
					{
						m = f->variable.array->size;
						a = sfprints("[%d]", f->variable.array->size);
					}
					else
						p = "*";
				}
				if (!f->structure.level)
					f->variable.structure = 0;
				if (f->variable.format.flags & CX_BUFFER)
				{
					n += sfprintf(sfstdout, "struct _%s_%s_buf_s", flat->basemeth->name, flat->meth.name);
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s;", p, f->variable.name, a);
				}
				else if ((f->variable.format.flags & CX_STRING) || !(f->physical.format.flags & CX_BINARY))
				{
					n += sfprintf(sfstdout, "char");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s", p, f->variable.name, a);
					if (f->physical.format.width > 1)
						n += sfprintf(sfstdout, "[%u];", f->physical.format.width);
					else
						n += sfprintf(sfstdout, ";");
				}
				else
				{
					if (f->variable.format.flags & CX_FLOAT)
						n += sfprintf(sfstdout, "_ast_flt");
					else
					{
						if ((f->variable.format.flags & CX_UNSIGNED) || !streq((char*)f->variable.type, "number"))
							n += sfprintf(sfstdout, "unsigned ");
						n += sfprintf(sfstdout, "_ast_int");
					}
					n += sfprintf(sfstdout, "%u_t", f->physical.format.width);
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s;", p, f->variable.name, a);
					if ((offset % f->physical.format.width) && !cxisvoid(f->variable.type))
					{
						s = "(MISALIGNED) ";
						if (disc->errorf)
							(*disc->errorf)(NiL, disc, 1, "%s: field size %u offset %u is not aligned", f->variable.name, f->physical.format.width, offset);
					}
				}
				offset += m * f->physical.format.width;
				sfputc(sfstdout, '\n');
				if (f->physical.format.delimiter >= 0)
				{
					n = sfprintf(sfstdout, "char");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "_delimiter_%d;", ++pad);
					tabs(sfstdout, n, 40);
					offset += 1;
					tmp[0] = f->physical.format.delimiter;
					tmp[1] = 0;
					sfprintf(sfstdout, "/* delimiter '%s' */\n", fmtesc(tmp));
				}
				if (!f->structure.next)
				{
					v = &f->variable;
					if (v->structure)
						while (!v->structure->next && (v = v->structure->parent) && v->structure)
						{
							v->structure->size = offset - v->structure->size;
							if (v->array)
							{
								if (v->array->size)
								{
									offset += (v->array->size - 1) * v->structure->size;
									sfprintf(sfstdout, "} %s[%d]", v->name, v->array->size);
								}
								else
									sfprintf(sfstdout, "} *%s", v->name);
								if (v->array->variable)
									sfprintf(sfstdout, " /*%s*/", v->array->variable->name);
								sfprintf(sfstdout, ";\n");
							}
							else
								sfprintf(sfstdout, "} %s;\n", v->name);
						}
				}
			}
			if (flat->binary && (fixed = offset % 8))
			{
				fixed = 8 - fixed;
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 1, "%s: record has %u unused pad byte%s", flat->meth.name, fixed, fixed == 1 ? "" : "s");
			}
			sfprintf(sfstdout, "%s\t%lu\t%u\t%u\tstruct\n", ".", offset, 0, 0);
			break;
		case 's':
			if (!flat->fixed)
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 2, "variable width record structs not supported");
				return -1;
			}
			for (f = flat->fields; f; f = f->next)
				if (f->variable.format.flags & CX_BUFFER)
				{
					sfprintf(sfstdout, "/* buffer field info */\n");
					sfprintf(sfstdout, "struct _%s_%s_buf_s\n", flat->basemeth->name, flat->meth.name);
					sfprintf(sfstdout, "{\n");
					n = sfprintf(sfstdout, "unsigned _ast_int2_t");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "offset;");
					tabs(sfstdout, n, 40);
					sfprintf(sfstdout, "/* buffer data _HEAP_ offset */\n");
					n = sfprintf(sfstdout, "unsigned _ast_int2_t");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "size;");
					tabs(sfstdout, n, 40);
					sfprintf(sfstdout, "/* buffer data size */\n");
					sfprintf(sfstdout, "};\n");
					break;
				}
			if (flat->meth.description)
				sfprintf(sfstdout, "/* %s */\n", flat->meth.description);
			sfprintf(sfstdout, "struct _%s_%s_s\n", flat->basemeth->name, flat->meth.name);
			sfprintf(sfstdout, "{\n");
			offset = 0;
			pad = 0;
			for (f = flat->fields; f; f = f->next)
			{
				if (f->structure.members)
				{
					f->structure.size = offset;
					sfprintf(sfstdout, "struct\n{\n");
					continue;
				}
				a = p = s = "";
				m = 1;
				n = 0;
				if (f->variable.array)
				{
					if (f->variable.array->size)
					{
						m = f->variable.array->size;
						a = sfprints("[%d]", f->variable.array->size);
					}
					else
						p = "*";
				}
				if (f->structure.level)
					n += sfprintf(sfstdout, "/*%02d*/", f->structure.level);
				else
					f->variable.structure = 0;
				if (f->variable.format.flags & CX_BUFFER)
				{
					n += sfprintf(sfstdout, "struct _%s_%s_buf_s", flat->basemeth->name, flat->meth.name);
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s;", p, f->variable.name, a);
				}
				else if ((f->variable.format.flags & CX_STRING) || !(f->physical.format.flags & CX_BINARY))
				{
					n += sfprintf(sfstdout, "char");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s", p, f->variable.name, a);
					if (f->physical.format.width > 1)
						n += sfprintf(sfstdout, "[%u];", f->physical.format.width);
					else
						n += sfprintf(sfstdout, ";");
				}
				else
				{
					if (f->variable.format.flags & CX_FLOAT)
						n += sfprintf(sfstdout, "_ast_flt");
					else
					{
						if ((f->variable.format.flags & CX_UNSIGNED) || !streq((char*)f->variable.type, "number"))
							n += sfprintf(sfstdout, "unsigned ");
						n += sfprintf(sfstdout, "_ast_int");
					}
					n += sfprintf(sfstdout, "%u_t", f->physical.format.width);
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "%s%s%s;", p, f->variable.name, a);
					if ((offset % f->physical.format.width) && !cxisvoid(f->variable.type))
					{
						s = "(MISALIGNED) ";
						if (disc->errorf)
							(*disc->errorf)(NiL, disc, 1, "%s: field size %u offset %u is not aligned", f->variable.name, f->physical.format.width, offset);
					}
				}
				offset += m * f->physical.format.width;
				if (*s || f->variable.description)
				{
					tabs(sfstdout, n, 40);
					sfprintf(sfstdout, "/* %s%s */\n", s, f->variable.description ? f->variable.description : "");
				}
				else
					sfputc(sfstdout, '\n');
				if (f->physical.format.delimiter >= 0)
				{
					n = sfprintf(sfstdout, "char");
					n = tabs(sfstdout, n, 24);
					n += sfprintf(sfstdout, "_delimiter_%d;", ++pad);
					tabs(sfstdout, n, 40);
					offset += 1;
					tmp[0] = f->physical.format.delimiter;
					tmp[1] = 0;
					sfprintf(sfstdout, "/* delimiter '%s' */\n", fmtesc(tmp));
				}
				if (!f->structure.next)
				{
					v = &f->variable;
					if (v->structure)
						while (!v->structure->next && (v = v->structure->parent) && v->structure)
						{
							v->structure->size = offset - v->structure->size;
							if (v->array)
							{
								if (v->array->size)
								{
									offset += (v->array->size - 1) * v->structure->size;
									sfprintf(sfstdout, "} %s[%d]", v->name, v->array->size);
								}
								else
									sfprintf(sfstdout, "} *%s", v->name);
								if (v->array->variable)
									sfprintf(sfstdout, " /*%s*/", v->array->variable->name);
								sfprintf(sfstdout, ";\n");
							}
							else
								sfprintf(sfstdout, "} %s;\n", v->name);
						}
				}
			}
			sfprintf(sfstdout, "};\n");
			if (flat->binary && (fixed = offset % 8))
			{
				fixed = 8 - fixed;
				sfprintf(sfstdout, "/* struct has %u unused pad byte%s */\n", fixed, fixed == 1 ? "" : "s");
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, 1, "%s: record has %u unused pad byte%s", flat->meth.name, fixed, fixed == 1 ? "" : "s");
			}
			sfprintf(sfstdout, "/* sizeof(struct _%s_%s_s)==%u */\n", flat->basemeth->name, flat->meth.name, offset);
			break;
		}
		if (flat->list || flat->prototype)
			exit(0);
		flat->getf = flatget;
		if (flat->binary && flat->fixed && !(flat->test & 0x0010))
		{
			offset = 0;
			for (f = flat->fields; f; f = f->next)
			{
				if ((f->variable.format.flags & CX_BINARY) && ((offset % f->physical.format.width) || (f->physical.format.width & (f->physical.format.width - 1)) || f->physical.format.width > 8) || f->width)
					break;
				offset += f->physical.format.width;
			}
			if (!f && !(offset % 8))
				flat->getf = flatgetbinary;
		}
		if (flat->record || (flat->fixed % 8))
			flat->getf = flatget;
		flat->valsiz = 64;
		if (!(flat->valbuf = newof(0, char, flat->valsiz, 0)))
		{
			if (disc->errorf)
				(*disc->errorf)(dss, disc, ERROR_SYSTEM|2, "out of space");
			return -1;
		}
		if (!flat->fixed || flat->record)
			flat->variable = 0;
		if (flat->variable || (dss->flags & DSS_FORCE))
			flat->force = 1;
	}
	return 0;
}

static Dssmeth_t method =
{
	"flat",
	"fixed-width and/or field-delimited flat file data",
	CXH,
	flatmeth,
	flatopen,
	0,
	0,
	0,
	0,
	0,
	0,
	0,
	DSS_BASE
};

static const char flatten_usage[] =
"[-1ls5P?\n@(#)$Id: dss flatten query (AT&T Research) 2005-05-09 $\n]"
USAGE_LICENSE
"[+PLUGIN?\findex\f]"
"[+DESCRIPTION?Flatten input data to match flat method \bschema\b.]"
"[e:emptyspace?Write empty field values as one \bspace\b character.]"
"\n"
"\nschema\n"
"\n"
;

/*
 * get source field value
 */

static Cxvalue_t*
flattenget(register Flatten_t* flatten, Cxvariable_t* var, void* data)
{
	Cxinstruction_t	x;
	Cxreference_t*	ref;

	if (ref = var->reference)
	{
		x.data.variable = ref->variable;
		if ((*flatten->cx->getf)(flatten->cx, &x, &flatten->value, NiL, NiL, data, flatten->cx->disc))
			return &nullval;
		while (ref = ref->next)
		{
			x.data.variable = ref->variable;
			if ((*ref->member->getf)(flatten->cx, &x, &flatten->value, NiL, NiL, NiL, flatten->cx->disc))
				return &nullval;
		}
	}
	else
	{
		x.data.variable = var;
		if ((*flatten->cx->getf)(flatten->cx, &x, &flatten->value, NiL, NiL, data, flatten->cx->disc))
			return &nullval;
	}
	return &flatten->value.value;
}

/*
 * flattenget() with cxnum2str()
 */

static Cxvalue_t*
flattennum2str(register Flatten_t* flatten, Cxvariable_t* var, void* data)
{
	Cxvalue_t*	val;

	if ((val = flattenget(flatten, var, data)) != &nullval)
	{
		if (cxnum2str(flatten->cx, &var->format, (Cxinteger_t)val->number, &val->string.data))
			return &nullval;
		val->string.size = strlen(val->string.data);
	}
	return val;
}

/*
 * flattenget() with cxstr2num()
 */

static Cxvalue_t*
flattenstr2num(register Flatten_t* flatten, Cxvariable_t* var, void* data)
{
	Cxvalue_t*	val;
	Cxunsigned_t	u;

	if ((val = flattenget(flatten, var, data)) != &nullval)
	{
		if (cxstr2num(flatten->cx, &var->format, val->string.data, val->string.size, &u))
			return &nullval;
		val->number = (Cxinteger_t)u;
	}
	return val;
}

/*
 * get nullval field value
 */

static Cxvalue_t*
flattennull(Flatten_t* flatten, Cxvariable_t* var, void* data)
{
	return &nullval;
}

/*
 * flatten query begin
 */

static int
flatten_beg(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	char**		argv = (char**)data;
	int		errors = error_info.errors;
	int		emptyspace = 0;
	char*		schema;
	Dssmeth_t*	meth;
	Flatten_t*	flatten;
	Field_t*	f;
	Field_t*	g;
	Vmalloc_t*	vm;
	int		offset;
	int		fixed;

	for (;;)
	{
		switch (optget(argv, flatten_usage))
		{
		case 'e':
			emptyspace = 1;
			continue;
		case '?':
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_USAGE|4, "%s", opt_info.arg);
			continue;
		case ':':
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s", opt_info.arg);
			continue;
		}
		break;
	}
	if (error_info.errors > errors)
		return -1;
	argv += opt_info.index;
	if (!(schema = *argv++) || *argv)
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_USAGE|4, "%s", optusage(NiL));
		return -1;
	}
	if (!(vm = vmopen(Vmdcheap, Vmlast, 0)) || !(flatten = vmnewof(vm, 0, Flatten_t, 1, 0)))
	{
		if (vm)
			vmclose(vm);
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
#if _HUH_2010_05_28
	if (!strchr(schema, ':'))
		schema = sfprints("%s:%s", *(char**)data, schema);
#endif
	flatten->vm = vm;
	flatten->emptyspace = emptyspace;
	if (!(meth = dssmeth(schema, disc)) || !(flatten->dss = dssopen(0, 0, disc, meth)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: cannot open conversion method", schema);
		goto bad;
	}
	flatten->cx = flatten->dss->cx;
	if (!flatten->cx->getf && !(flatten->cx->getf = cxcallout(flatten->cx, CX_GET, flatten->cx->state->type_void, flatten->cx->state->type_void, flatten->cx->disc)))
	{
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 3, "cx CX_GET callout must be defined");
		goto bad;
	}
	flatten->getf = flatten->cx->getf;
	flatten->flat = (Flat_t*)flatten->dss->meth->data;
	offset = 0;
	fixed = flatten->flat->fixed;
	for (f = flatten->flat->fields; f; f = f->next)
	{
		f->flattengetf = flattenget;
		if (!(f->flatten = (Cxvariable_t*)dtmatch(cx->variables, f->variable.name)))
		{
			f->flattengetf = flattennull;
			if (!cxisvoid(f->variable.type) && disc->errorf)
				(*disc->errorf)(NiL, disc, 1, "%s: field not in source record -- default value will be output", f->variable.name);
		}
		else if (f->flatten->format.map)
		{
			if (cxisnumber(f->variable.type) && cxisstring(f->flatten->type))
				f->flattengetf = flattenstr2num;
			else if (cxisstring(f->variable.type) && cxisnumber(f->flatten->type))
				f->flattengetf = flattennum2str;
		}
		if ((g = (Field_t*)dtmatch(cx->variables, f->variable.name)) && g->physical.format.code != f->physical.format.code && g->physical.format.code == CC_NATIVE)
			f->pam = ccmap(g->physical.format.code, f->physical.format.code);
		if (fixed && (f->variable.format.flags & CX_BINARY) && (offset % f->physical.format.width))
		{
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s: field size %u offset %u is not aligned", f->variable.name, f->physical.format.width, offset);
			goto bad;
		}
		offset += f->physical.format.width;
	}
	if (flatten->flat->binary && (offset %= 8))
	{
		offset = 8 - offset;
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, 2, "%s: record has %u unused pad byte%s", flatten->dss->meth->name, offset, offset == 1 ? "" : "s");
		goto bad;
	}
	if (!(flatten->file = dssfopen(flatten->dss, NiL, expr->op, DSS_FILE_WRITE, &flat_format)))
		goto bad;
	expr->op = flatten->file->io;
	expr->data = flatten;
	return 0;
 bad:
	if (flatten->file)
		dssfclose(flatten->file);
	if (flatten->dss)
		dssclose(flatten->dss);
	vmclose(vm);
	return -1;
}

/*
 * flatten query action
 */

static int
flatten_act(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	register Flatten_t*	flatten = (Flatten_t*)expr->data;
	register Field_t*	f;
	register unsigned char*	s;
	Cxvalue_t*		v;
	unsigned char*		b;
	unsigned char*		e;
	ssize_t			n;
	int			q;

	for (f = flatten->flat->fields; f; f = f->next)
	{
		v = (*f->flattengetf)(flatten, f->flatten, data);
		while ((n = (*f->physical.type->externalf)(flatten->cx, f->physical.type, NiL, &f->physical.format, v, flatten->flat->valbuf, flatten->flat->valsiz, disc)) > flatten->flat->valsiz)
		{
			n = roundof(n, 32);
			if (!(flatten->flat->valbuf = newof(flatten->flat->valbuf, char, n, 0)))
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
				return -1;
			}
			flatten->flat->valsiz = n;
		}
		if (n < 0)
			return -1;
		else if (n > 0)
		{
			if (f->pam)
				ccmapcpy(f->pam, flatten->flat->valbuf, flatten->flat->valbuf, n);
			if ((f->physical.format.flags & (CX_STRING|CX_BUFFER)) && f->physical.format.delimiter >= 0 && (f->physical.format.escape >= 0 || f->physical.format.quotebegin >= 0))
			{
				if (f->physical.format.flags & CX_QUOTEALL)
				{
					q = 1;
					sfputc(flatten->file->io, f->physical.format.quotebegin);
				}
				else
					q = 0;
				for (e = (s = b = (unsigned char*)flatten->flat->valbuf) + n; s < e; s++)
					if (*s == f->physical.format.delimiter || *s == f->physical.format.escape || *s == f->physical.format.quotebegin || *s == f->physical.format.quoteend)
					{
						if (f->physical.format.escape >= 0)
						{
							sfwrite(flatten->file->io, b, s - b);
							sfputc(flatten->file->io, f->physical.format.escape);
							sfputc(flatten->file->io, *s);
						}
						else if (*s == f->physical.format.delimiter)
						{
							if (q)
								continue;
							q = 1;
							sfwrite(flatten->file->io, b, s - b);
							sfputc(flatten->file->io, f->physical.format.quotebegin);
							sfputc(flatten->file->io, *s);
						}
						else
						{
							sfwrite(flatten->file->io, b, s - b + 1);
							sfputc(flatten->file->io, *s);
							if (!q)
							{
								q = 1;
								sfputc(flatten->file->io, *s);
							}
						}
						b = s + 1;
					}
				if (q && !(f->physical.format.flags & CX_QUOTEALL))
				{
					q = 0;
					sfputc(flatten->file->io, f->physical.format.quoteend);
				}
				sfwrite(flatten->file->io, b, s - b);
				if (q)
					sfputc(flatten->file->io, f->physical.format.quoteend);
			}
			else
				sfwrite(flatten->file->io, flatten->flat->valbuf, n);
		}
		else if (flatten->emptyspace && f->physical.format.delimiter >= 0)
			sfputc(flatten->file->io, ' ');
		if (f->physical.format.delimiter >= 0)
			sfputc(flatten->file->io, f->physical.format.delimiter);
	}
	return 0;
}

/*
 * flatten query end
 */

static int
flatten_end(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	Flatten_t*	flatten = (Flatten_t*)expr->data;
	int		r;

	if (!flatten)
		return -1;
	expr->data = 0;
	r = 0;
	if (dssfclose(flatten->file))
		r = -1;
	if (dssclose(flatten->dss))
		r = -1;
	vmclose(flatten->vm);
	return r;
}

static Cxquery_t queries[] =
{
{ "flatten",	"query to flatten input data to flat schema format",
		CXH, flatten_beg, 0, flatten_act, flatten_end },
{0}
};

Dsslib_t dss_lib_flat =
{
	"flat",
	"flat method"
	"[-1ls5Pp0?\n@(#)$Id: dss flat method (AT&T Research) 2011-08-19 $\n]"
	USAGE_LICENSE,
	CXH,
	0,
	&method,
	0,
	0,
	0,
	0,
	&queries[0],
};