validate.c

/***********************************************************************
*                                                                      *
*               This software is part of the ast package               *
*          Copyright (c) 2003-2011 AT&T Intellectual Property          *
*                      and is licensed under the                       *
*                 Eclipse Public License, Version 1.0                  *
*                    by AT&T Intellectual Property                     *
*                                                                      *
*                A copy of the License is available at                 *
*          http://www.eclipse.org/org/documents/epl-v10.html           *
*         (with md5 checksum b35adb5213ca9657e911e9befb180842)         *
*                                                                      *
*              Information and Software Systems Research               *
*                            AT&T Research                             *
*                           Florham Park NJ                            *
*                                                                      *
*                 Glenn Fowler <[email protected]>                  *
*                                                                      *
***********************************************************************/
#pragma prototyped

static const char validate_usage[] =
"[+PLUGIN?\findex\f]"
"[+DESCRIPTION?The validate query validates the constraints of the"
"	\afield\a operands. If no operands are specified then all"
"	fields with constraints or maps are validated. A warning is"
"	printed if there are no fields with constraints or maps.]"
"[d:discard?Discard records containing invalid fields.]"
"[l:list?List the field constraints and exit.]"
"[s:summary?Print a summary after all records have been read.]"
"[r:repair?Repair invalid fields if possible.]"
"[v:verbose?Warn about each invalid field and the action taken.]"
"\n"
"\n[ field ... ]\n"
"\n"
;

#include <dsslib.h>
#include <ast_float.h>

struct Field_s; typedef struct Field_s Field_t;
struct Invalid_s; typedef struct Invalid_s Invalid_t;
struct State_s; typedef struct State_s State_t;

struct Field_s
{
	Field_t*	next;
	Cxvariable_t*	variable;
	Cxinteger_t	invalid;
	Cxinteger_t	discarded;
	Cxinteger_t	repaired;
};

struct Invalid_s
{
	Dtlink_t	link;
	Cxvalue_t	value;
	Cxvariable_t*	variable;
	Cxunsigned_t	count;
};

struct State_s
{
	Field_t*	field;
	Cxcallout_f	getf;
	Cxcallout_f	setf;
	Dt_t*		invalid;
	Dtdisc_t	invaliddisc;
	Vmalloc_t*	vm;
	unsigned char	discard;
	unsigned char	summary;
	unsigned char	verbose;
};

extern Dsslib_t		dss_lib_validate;

static void
number(Sfio_t* op, const char* label, Cxnumber_t n, Cxformat_t* format)
{
	sfprintf(op, " %s=", label);
	if (format->details)
	{
		if (((n >= 0) ? n : -n) < 1)
			n = 0;
		else if (n > FLTMAX_INTMAX_MAX)
			n = FLTMAX_INTMAX_MAX;
		else if (n < FLTMAX_INTMAX_MIN)
			n = FLTMAX_INTMAX_MIN;
		sfprintf(op, format->details, (Cxinteger_t)n);
	}
	else if (n == 0 || ((n >= 0) ? n : -n) >= 1 && n >= FLTMAX_INTMAX_MIN && n <= FLTMAX_UINTMAX_MAX && n == (Cxinteger_t)n)
		sfprintf(op, (format->flags & CX_UNSIGNED) ? "%llu" : "%lld", (Cxinteger_t)n);
	else
		sfprintf(op, "%1.15Lg", n);
}

static int
invalidcmp(Dt_t* dict, void* a, void* b, Dtdisc_t* disc)
{
	Invalid_t*	ap = (Invalid_t*)a;
	Invalid_t*	bp = (Invalid_t*)b;
	size_t		az;
	size_t		bz;
	int		r;

	if (!(r = strcmp(ap->variable->name, bp->variable->name)))
	{
		if (cxisstring(ap->variable->type) || cxisbuffer(ap->variable->type))
		{
			az = ap->value.buffer.size;
			bz = bp->value.buffer.size;
			if (!(r = memcmp(ap->value.buffer.data, bp->value.buffer.data, az < bz ? az : bz)))
			{
				if (az < bz)
					r = -1;
				if (az > bz)
					r = 1;
			}
		}
		else if (ap->value.number < bp->value.number)
			r = -1;
		else if (ap->value.number > bp->value.number)
			r = 1;
	}
	return r;
}

static int
validate_beg(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	char**			argv = (char**)data;
	int			errors = error_info.errors;
	char*			s;
	State_t*		state;
	Cxvariable_t*		variable;
	register Field_t*	field;
	Field_t*		lastfield;
	Cxconstraint_t*		constraint;
	int			all;
	int			list;
	Vmalloc_t*		vm;

	if (!(vm = vmopen(Vmdcheap, Vmlast, 0)) || !(state = vmnewof(vm, 0, State_t, 1, 0)))
	{
		if (vm)
			vmclose(vm);
		if (disc->errorf)
			(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
		return -1;
	}
	state->vm = vm;
	list = 0;
	sfprintf(cx->buf, "%s%s", strchr(dss_lib_validate.description, '['), validate_usage);
	s = sfstruse(cx->buf);
	for (;;)
	{
		switch (optget(argv, s))
		{
		case 'd':
			state->discard = 1;
			continue;
		case 'l':
			list = 1;
			continue;
		case 'r':
			if (!(state->setf = cxcallout(cx, CX_SET, cx->state->type_void, cx->state->type_void, cx->disc)))
			{
				if (cx->disc->errorf)
					(*cx->disc->errorf)(NiL, cx->disc, 3, "reair requires CX_SET callout");
				return -1;
			}
			continue;
		case 's':
			state->summary = 1;
			continue;
		case 'v':
			state->summary = state->verbose = 1;
			continue;
		case '?':
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, ERROR_USAGE|4, "%s", opt_info.arg);
			else
				return -1;
			continue;
		case ':':
			if (disc->errorf)
				(*disc->errorf)(NiL, disc, 2, "%s", opt_info.arg);
			else
				return -1;
			continue;
		}
		break;
	}
	if (error_info.errors > errors)
		goto bad;
	argv += opt_info.index;
	if (all = !*argv)
		variable = 0;
	do
	{
		if (all)
		{
			if (!(variable = (Cxvariable_t*)(variable ? dtnext(cx->fields, variable) : dtfirst(cx->fields))))
				break;
		}
		else if (!(variable = cxvariable(cx, *argv, NiL, disc)))
			goto bad;
		if (variable->format.constraint || variable->format.map)
		{
			if (!(field = vmnewof(vm, 0, Field_t, 1, 0)))
			{
				if (disc->errorf)
					(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
				goto bad;
			}
			field->variable = variable;
			if (state->field)
				lastfield = lastfield->next = field;
			else
				lastfield = state->field = field;
		}
	} while (all || *++argv);
	if (!state->field && disc->errorf)
		(*disc->errorf)(NiL, disc, 1, "no field has constraints or maps");
	if (list)
	{
		for (field = state->field; field; field = field->next)
		{
			sfprintf(expr->op, "%16s", field->variable->name);
			if (field->variable->format.map)
				sfprintf(expr->op, " map");
			if (constraint = field->variable->format.constraint)
			{
				if (constraint->name)
					sfprintf(expr->op, " name=%s", constraint->name);
				if (constraint->constraintf)
					sfprintf(expr->op, " external");
				if (cxisnumber(field->variable->type))
				{
					if (constraint->def)
						number(expr->op, "default", constraint->def->number, &field->variable->format);
					if (constraint->min)
						number(expr->op, "min", constraint->min->number, &field->variable->format);
					if (constraint->max)
						number(expr->op, "max", constraint->max->number, &field->variable->format);
				}
				else if (cxisstring(field->variable->type) && constraint->def)
					sfprintf(expr->op, " default=\"%-.*s\"", constraint->def->string.size, constraint->def->string.data);
				if (constraint->expression)
					sfprintf(expr->op, " expression=\"%s\"", constraint->expression);
				if (constraint->pattern)
					sfprintf(expr->op, " pattern=\"%s\"", constraint->pattern);
			}
			sfprintf(expr->op, "\n");
		}
		goto bad;
	}
	if (!(state->getf = cxcallout(cx, CX_GET, cx->state->type_void, cx->state->type_void, cx->disc)))
	{
		if (cx->disc->errorf)
			(*cx->disc->errorf)(NiL, cx->disc, 3, "validation requires CX_GET callout");
		goto bad;
	}
	if (!state->verbose)
	{
		state->invaliddisc.comparf = invalidcmp;
		if (!(state->invalid = dtnew(vm, &state->invaliddisc, Dtoset)))
		{
			if (cx->disc->errorf)
				(*cx->disc->errorf)(NiL, cx->disc, 3, "validation requires CX_GET callout");
			goto bad;
		}
	}
	expr->data = state;
	return 0;
 bad:
	vmclose(vm);
	return -1;
}

static int
validate_sel(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	register State_t*	state = (State_t*)expr->data;
	register Field_t*	field;
	register Cxconstraint_t*constraint;
	Cxoperand_t		o;
	Cxinstruction_t		x;
	Invalid_t		key;
	Invalid_t*		ip;
	size_t			n;

	for (field = state->field; field; field = field->next)
	{
		x.data.variable = field->variable;
		if ((*state->getf)(cx, &x, &o, NiL, NiL, data, disc))
			return -1;
		if (field->variable->format.map)
		{
			if (cxisstring(field->variable->type))
			{
				if (cxstr2num(cx, &field->variable->format, o.value.string.data, o.value.string.size, NiL))
				{
					if (state->verbose && disc->errorf)
						(*disc->errorf)(NiL, disc, 1, "%s%s: %-.*s: unknown map name", cxlocation(cx, data), field->variable->name, o.value.string.size, o.value.string.data);
					goto invalid;
				}
			}
			else if (cxisnumber(field->variable->type))
			{
				if (cxnum2str(cx, &field->variable->format, (Cxinteger_t)o.value.number, NiL))
				{
					if (state->verbose && disc->errorf)
						(*disc->errorf)(NiL, disc, 1, "%s%s: %I*d: unknown map value", cxlocation(cx, data), field->variable->name, sizeof(Cxinteger_t), (Cxinteger_t)o.value.number);
					goto invalid;
				}
			}
		}
		if (constraint = field->variable->format.constraint)
		{
			if (constraint->constraintf)
				;
			if (cxisnumber(field->variable->type))
			{
				if (constraint->min && o.value.number < constraint->min->number)
				{
					if (state->verbose && disc->errorf)
						(*disc->errorf)(NiL, disc, 1, "%s%s: %1.15Lg violates min constraint %1.15Lg", cxlocation(cx, data), field->variable->name, o.value.number, constraint->min->number);
					goto invalid;
				}
				if (constraint->max && o.value.number > constraint->max->number)
				{
					if (state->verbose && disc->errorf)
						(*disc->errorf)(NiL, disc, 1, "%s%s: %1.15Lg violates max constraint %1.15Lg", cxlocation(cx, data), field->variable->name, o.value.number, constraint->max->number);
					goto invalid;
				}
			}
			if (constraint->expression)
				;
			if (constraint->pattern)
				;
		}
		continue;
	invalid:
		if (state->invalid)
		{
			key.variable = field->variable;
			key.value = o.value;
			if (!(ip = (Invalid_t*)dtsearch(state->invalid, &key)))
			{
				n = cxsize(field->variable->type, &o.value);
				if (!(ip = vmnewof(state->vm, 0, Invalid_t, 1, n)))
				{
					if (disc->errorf)
						(*disc->errorf)(NiL, disc, ERROR_SYSTEM|2, "out of space");
					return -1;
				}
				*ip = key;
				ip->value = o.value;
				if (n)
				{
					ip->value.buffer.data = (void*)(ip + 1);
					memcpy(ip->value.buffer.data, o.value.buffer.data, n);
				}
				dtinsert(state->invalid, ip);
				ip->count = 0;
			}
			ip->count++;
		}
		if (state->setf && constraint && constraint->def)
		{
			o.type = field->variable->type;
			o.value = *constraint->def;
			if ((*state->setf)(cx, &x, &o, &o, NiL, data, disc))
				return -1;
			field->repaired++;
		}
		else if (state->discard)
		{
			field->discarded++;
			return 0;
		}
		else
			field->invalid++;
	}
	return 1;
}

static int
validate_end(Cx_t* cx, Cxexpr_t* expr, void* data, Cxdisc_t* disc)
{
	register State_t*	state = (State_t*)expr->data;
	register Field_t*	field;
	Invalid_t*		ip;
	Cxoperand_t		val;
	int			heading;

	if (state->summary)
	{
		heading = 1;
		if (state->invalid && dtsize(state->invalid))
		{
			heading = 0;
			sfprintf(expr->op, "%16s  %11s  %s\n", "FIELD", "COUNT", "VALUE");
			for (ip = (Invalid_t*)dtfirst(state->invalid); ip; ip = (Invalid_t*)dtnext(state->invalid, ip))
			{
				val.type = ip->variable->type;
				val.value = ip->value;
				if (!cxcast(cx, &val, NiL, cx->state->type_string, NiL, NiL))
					sfprintf(expr->op, "%16s  %11I*u  %*.*s\n", ip->variable->name, sizeof(ip->count), ip->count, val.value.string.size, val.value.string.size, val.value.string.data);
			}
		}
		if (!heading)
		{
			heading = 1;
			sfprintf(expr->op, "\n");
		}
		for (field = state->field; field; field = field->next)
			if (field->invalid || field->discarded || field->repaired)
			{
				if (heading)
				{
					heading = 0;
					sfprintf(expr->op, "%16s  %11s %11s %11s\n", "FIELD", "INVALID", "DISCARDED", "REPAIRED");
				}
				sfprintf(expr->op, "%16s  %11I*u %11I*u %11I*u\n", field->variable->name, sizeof(field->invalid), field->invalid, sizeof(field->discarded), field->discarded, sizeof(field->repaired), field->repaired);
			}
	}
	vmclose(state->vm);
	return 0;
}

static Cxquery_t	queries[] =
{
	{
		"validate",
		"validate field value constraints",
		CXH,
		validate_beg,
		validate_sel,
		0,
		validate_end
	},
	{0}
};

Dsslib_t		dss_lib_validate =
{
	"validate",
	"validate query"
	"[-1lms5P?\n@(#)$Id: dss validate query (AT&T Research) 2003-04-05 $\n]"
	USAGE_LICENSE,
	CXH,
	0,
	0,
	0,
	0,
	0,
	0,
	&queries[0]
};