spOSMroute/mapsys/map_searchhash.c

/***************************************************************************
 *            map_searchhash.c
 *
 *  Copyright  2010 - Steffen Pohle
 *  steffen@gulpe.de
 ****************************************************************************/

/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */

#ifdef HAVE_LOCALE_H
  #include <locale.h>
#endif
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>


#include "osmroute.h"
#include "map.h"
#include "memoryleak.h"
#include "system.h"


int map_search_fd = 0;
int map_search_disabled = 0;

/****************************************************************************/
struct map_search_data map_search_line2data (char *line);


/*****************************************************************************
 *
 * searching and filemanagment for the search files, this also includes
 * saving and reading files. All saved data will be in text format.
 * Country and Name will always be converted to lowercase text
 */
void map_search_get_filename (char *fn, int len) {
#ifdef HAVE_LOCALE_H
	setlocale(LC_ALL, "C");
	snprintf (fn, len, "%s/search.mapidx", cfg.mappath);
	setlocale(LC_ALL, NULL);
#else
	snprintf (fn, len, "%s/search.mapidx", cfg.mappath);
#endif	
};


void map_search_open () {
	char fn[LEN_FILENAME];

	if (map_search_disabled) return;
	if (map_search_fd <= 0) {
		map_search_get_filename (fn, LEN_FILENAME);
		map_search_fd = open (fn, O_RDWR | O_CREAT, 0644);
	}
};


/****************************************************************************
 * convert between line and structure
 */
struct map_search_data map_search_line2data (char *line) {
	struct map_search_data msd;
	char *p = line;

	strncpy (msd.name, p, MAP_SEARCHLINE_LEN);
	p = p + strlen(p) + 1;

	strncpy (msd.country, p, MAP_SEARCHLINE_LEN);
	p = p + strlen(p) + 1;

	sscanf (p, "%lld %f %f", &msd.id, &msd.lon, &msd.lat);

	return msd;
};


/**************************************************************************
 * add a new entry to the search list.
 * be carefull searching works only fine on sorted lists. So after adding
 * lines to the structure it has to be sorted again.
 */
void map_search_add (char *country, char *name, unsigned long long int id, float lon, float lat) {
	char dataline[MAP_SEARCHLINE_LEN];
	char *n, *c, *cp;
	char nullstring[2] = "\0\0";

	if (map_search_disabled) return;
//	d_printf ("map_search_add id:%lld name:'%s'", id, name);
	
	if (name == NULL) n = nullstring;
	else n = name;
	if (country == NULL) c = nullstring;
	else c = country;

	if (map_search_fd <= 0) map_search_open ();
	
	if (map_search_fd) {
		lseek (map_search_fd, 0, SEEK_END);
		/* copy country */
		cp = dataline;

		strncpy (cp, n, MAP_SEARCHLINE_LEN-(cp-dataline));
		cp = cp + strlen (cp); cp[0] = 0; cp++;

		strncpy (cp, c, MAP_SEARCHLINE_LEN-(cp-dataline));
		cp = cp + strlen (cp); cp[0] = 0; cp++;

		snprintf (cp, MAP_SEARCHLINE_LEN-(cp-dataline), "%lld %f %f\n", id, lon, lat);
		cp = cp + strlen (cp);
		
		write (map_search_fd, dataline, cp-dataline);
	}
};


/*
 * just reset all data
 */
void map_search_init () {
	if (map_search_disabled) return;
	if (map_search_fd) close (map_search_fd);
	map_search_fd = 0;
};


/**************************************************************************
 * sort the search.mapidx file in alphabetic order
 * - read full file into memory
 * - create pointer array to all elements
 * - sort the pointers in alphabetic order
 * - delete old file, save new file ignore double id entrys
 */
#define MAP_SEARCH_SORT_BLOCK 4096
struct maps_sort {
	char **larray;
	int max;
	int cnt;
};


void swap (char **data1, char **data2) {
	char *tmp = *data1;
	*data1 = *data2;
	*data2 = tmp;
};


struct maps_sort map_search_sort_checkmem (struct maps_sort sdat, int size) {
	struct maps_sort ns = { NULL, 0, 0};
	int i;

	if (size == 0) {
		if (sdat.larray) ml_free (sdat.larray);
		return ns;
	}
	else if (size+1 >= sdat.max) {
		ns.max = size + MAP_SEARCH_SORT_BLOCK;
		ns.larray = (char **) ml_malloc (sizeof (char *) * ns.max);
		for (i = 0; i < sdat.cnt; i++) 
			ns.larray[i] = sdat.larray[i];
		ns.cnt = sdat.cnt;
		ml_free (sdat.larray);

		return ns;
	}
	else return sdat;
};

/*****************************************************************************
 * bubblesort 
 */
void map_search_sort_alg1 (struct maps_sort *data) {
	int i, j;
	char text[256];

	for (j = 0; j < data->cnt; j++) {
		if (j > 0 && data->larray[j]) {
			i = j-1;
			while (i >= 0 && (data->larray[i] == NULL || strcasecmp (data->larray[i+1], data->larray[i]) > 0)) {
				swap (&data->larray[i+1], &data->larray[i]);
				i--;
			}
		}
		if (j % 100 == 0) {
			sprintf (text, _("Sorting search.mapidx %d/%d"), j, data->cnt);
			app_status (-1, text, 0.0);
			main_wnd_update ();
		}			    
	}
};


/*****************************************************************************
 * quicksort 
 */

/*
 * quicksortstack - needed for storing unsorted fields.
 */
struct maps_qsort {
	int elements;
	int start;
};

struct maps_qsort *maps_qs_data = NULL;	/* pointer to stack array 			*/
int maps_qs_size = 0;					/* allocated elements on stack		*/
int maps_qs_pos = 0;					/* current position on the stack	*/


/* push data to the quicksort stack */
void maps_qsort_push (struct maps_qsort *data) {
	int i;
	struct maps_qsort *tmp;
	
	if (maps_qs_size <= maps_qs_pos+1) {
		maps_qs_size += 1024;
		tmp = ml_malloc (sizeof (struct maps_qsort)*maps_qs_size);
		for (i = 0; i < maps_qs_pos; i++) tmp[i] = maps_qs_data[i];
		ml_free (maps_qs_data);
		maps_qs_data = tmp;
	}

	maps_qs_data[maps_qs_pos] = *data;
	maps_qs_pos++;
};

/* pop data form the quicksort stack */
void maps_qsort_pop (struct maps_qsort *data) {
	if (maps_qs_pos > 0) {
		maps_qs_pos--;
		*data = maps_qs_data[maps_qs_pos];
	}
	if (maps_qs_pos == 0 && maps_qs_size > 0) {
		ml_free (maps_qs_data);
		maps_qs_data = NULL;
		maps_qs_size = 0;
	}
};

/*
 * quicksort routine:
 * to prevent too many compares in case we have already sorted data we
 * check if the data is already sorted.
 */
void map_search_sort_alg2 (struct maps_sort *data) {
	int i, j, to = 0;
	char text[256];
	struct maps_qsort qsort, left, right;
	int pivot;
	
	if (data->cnt < 2) return; /* only one element --> return */

	/* select full data block */
	qsort.elements = data->cnt;
	qsort.start = 0;

	/* 
	 * loop until we have no data on the quicksort stack 
	 */
	do {
		/* 
		 * select first element as pivot to compare against 
		 * pivot is the selected element inside the block 
		 * i - element from left (smaller as pivot)
		 * j - element from right (larger as pivot)
		 */
		pivot = 0;
		i = 1;
		j = qsort.elements-1;
		/* make sure we have always mixed up lists.. */
		if (qsort.elements > 50) swap (&data->larray[qsort.start], &data->larray[qsort.start+qsort.elements/2]);

		do {
			/*
			 * go to right until we found something larger as pivot
			 */
			while (i < j && (strcasecmp (data->larray[qsort.start+pivot], data->larray[qsort.start+i]) >= 0 || pivot == i)) {
				i++;
			}
			
			/*
			 * go to the left until we found something smaller as pivot
			 */
			while (i < j && (strcasecmp (data->larray[qsort.start+pivot], data->larray[qsort.start+j]) < 0 || pivot == j)) {
				j--;
			}

			/*
			 * still i < j: swap i<-->j , swap i<-->pivot
			 */
			if (i < j) {
				swap (&data->larray[qsort.start+j], &data->larray[qsort.start+i]);
				swap (&data->larray[qsort.start+i], &data->larray[qsort.start+pivot]);
				pivot = i;
				i++;
				j--;
			}

			/*
			 * finished? but check in case the last elements are not in order
			 */
			if (i == j) {
				/* i < pivot: swap pivot<-->i */
				if (strcasecmp (data->larray[qsort.start+pivot], data->larray[qsort.start+i]) >= 0) {
					swap (&data->larray[qsort.start+i], &data->larray[qsort.start+pivot]);
					pivot = i;
				}
				/* i-1 < pivot: swap pivot<-->i-1 */
				else if (strcasecmp (data->larray[qsort.start+pivot], data->larray[qsort.start+i]-1) >= 0) {
					swap (&data->larray[qsort.start+i-1], &data->larray[qsort.start+pivot]);
					pivot = i-1;
				}
			}
	
			if (i == pivot) i++;
			if (j == pivot) j--;
		} while (i < j);

		/*
		 * split block in two and put them on the quicksort stack
		 */
		left.start = qsort.start;
		left.elements = pivot;
		right.start = pivot+qsort.start+1;
		right.elements = qsort.elements-pivot-1;
		/* only push if it is nessesary (not clean, or more then one element) */
		if (left.elements > 1) maps_qsort_push (&left);
		if (right.elements > 1) maps_qsort_push (&right);

		if (maps_qs_pos > 0) maps_qsort_pop (&qsort);
		else qsort.elements = 0;

		/* display update on the screen */
		if (to++ < 0 || to > 250) {
			sprintf (text, _("sorting search.mapidx %d"), maps_qs_pos);
			app_status (-1, text, 0.0);
			main_wnd_update ();
			to = 0;
		}

	} while (qsort.elements > 1);
};


void map_search_sort () {
	char fn_src[LEN_FILENAME];
	char text[256];
	int fd_src;
	char *fdata = NULL;
	char *c, *ci;
	struct maps_sort data = {NULL, 0, 0};
	struct map_search_data msd;
	struct stat fs;
	int i;
	size_t rs;
	unsigned long long int lid;
	
	/* close open files - reset and disable searching */
	d_printf ("map_search_sort entering");
#ifdef HAVE_LOCALE_H
	setlocale(LC_ALL, "C");
#endif
	map_search_init ();
	map_search_disabled = 1;
	data.max = data.cnt = 0;
	
	/* get filenames */
	map_search_get_filename (fn_src, LEN_FILENAME);
	d_printf ("map_search_sort: src:%s", fn_src);

	/* size of file and read file into memory */
	if (stat(fn_src, &fs) != 0) {
		d_printf ("map_search_sort: error on stat %s", fn_src);
		map_search_disabled = 0;
		return;
	}

	d_printf ("map_search_sort: read %ld of data", fs.st_size);
	fdata = ml_malloc (fs.st_size);
#if defined(__MINGW32CE__) || defined(_WIN32_WCE) || defined(__MINGW32__)
	fd_src = open (fn_src, O_RDWR | _O_BINARY);
#else
	fd_src = open (fn_src, O_RDWR);
#endif
	rs = read (fd_src, fdata, fs.st_size);
	d_printf ("                 got %ld", fs.st_size, rs);
	close (fd_src);

	/*
	 * create pointers of all new lines
	 */
	for (i = 0, c = fdata, ci = fdata; ci < fdata + rs; ci++) {
		if (i > 1 && (*ci == '\n' || *ci == '\r')) {
			if (*c != '\n' && *c != '\r') { /* save line */
				if (data.cnt+1 >= data.max) data = map_search_sort_checkmem (data, data.cnt+1024);
				data.larray[data.cnt] = c;
				data.cnt++;
				c = ci;
				i = 0;
			}
		}
		else { /* reset line */
			if (*ci == '\0') i++;
			if (*c == '\n' || *c == '\r')
				c = ci;
		}
	}
	d_printf ("map_search_sort: found %d entrys", data.cnt);

	/* sorting */
	map_search_sort_alg2 (&data);
	
	map_search_disabled = 0;
	d_printf ("map_search_sort: finished sorting.");
	lid = 0;
	unlink (fn_src);
	for (i = 0; i < data.cnt; i++) {
		if (data.larray[i]) {
			msd = map_search_line2data (data.larray[i]);
			if (lid != msd.id) {	/* ignore double entries */
				map_search_add (msd.country, msd.name, msd.id, msd.lon, msd.lat);
			}
			lid = msd.id;
			if (i % 10000 == 0) {
				sprintf (text, _("save search.mapidx %d"), i);
				app_status (-1, text, 0.0);
				main_wnd_update ();
			}
		}
		if (i > 0) if (strcasecmp (data.larray[i-1], data.larray[i]) > 0) {
			d_printf ("sorting went wrong. %d '%s' < %d '%s'", i-1, data.larray[i-1], i, data.larray[i]);
		}
	}
	d_printf ("map_search_sort: complete sorted");
	
#ifdef HAVE_LOCALE_H
	setlocale(LC_ALL, NULL);
#endif
	data = map_search_sort_checkmem (data, 0);
	ml_free (fdata);
};


/*****************************************************************************
 * searching routines: we will use the same way as searching on hashnodes 
 * during converting the planet.osm file.
 * check the center of the file and test fo larger or smaller value,
 * go into the larger or smaller value, check here also the center.. and so on.
 */

/*
 * read and convert data form position, if the correct flag is set it 
 * will check for the corrent start address and return it together with 
 * the size on file.
 */
static char maps_line[MAP_SEARCHLINE_LEN*4];
#warning maps_readpos.. read current position without checking 
#warning and return start of next line.

/* 
 * read an random position, check if we are at the start of valid data. If not
 * search for it.
 * parameter: pos - entry position inside file
 *            data - destination of found data
 *            size - size of data block
 * return: final start of current data block
 */
unsigned long int maps_readpos (unsigned long int pos, struct map_search_data *data, int *size) {
	size_t realpos;
	int readsize, start, end;

	if (map_search_fd <= 0) return -1;
	if (pos < MAP_SEARCHLINE_LEN*2) realpos = 0;
	else realpos = pos - MAP_SEARCHLINE_LEN*2;

	/* read data from file */
	lseek (map_search_fd, realpos, SEEK_SET);
	readsize = read (map_search_fd, maps_line, MAP_SEARCHLINE_LEN*4);
	if (readsize <= 0) return -1;

	/* search where the current line starts (go to the left) */
//	d_printf ("start at: pos:%ld  realpos:%d   start:%d", pos, realpos, pos-realpos);
	for (start = pos-realpos; start > 0; start--) {
//		d_printf ("  found:start:%-4d  %x   '%c'  delta:%d", start, maps_line[start], maps_line[start], realpos+start-pos);
		if (maps_line[start] == '\n') {
			start++;
			break;
		}
	}
	if (start < 0) {	/* underrun buffer? */
		d_printf ("maps_readpos: found no start. pos:%ld realpos:%lld", pos, realpos);
		return -1;
	}
//	d_printf ("  maps_readpos pos:%ld start:%d  delta:%d", pos, start, (realpos+start)-pos);

	/* search where the current line ends */
	for (end = start; end < readsize && maps_line[end] != '\n'; end++);
	if (end >= readsize) {
		d_printf ("maps_readpos: found no end.");
		return -1;
	}
		
	*data = map_search_line2data (maps_line+start);
	*size = end-start;
//	d_printf ("  size:%d", *size);
	
	return realpos+start;
};


int map_search (char *name, struct map_search_data *data, int datasize) {
	uint64_t blocksize, blockstart, realstart;
	struct stat buf;
	int i, j, cmpres, k;

	if (map_search_disabled || strlen (name) == 0) return 0;
	if (map_search_fd <= 0) map_search_open ();

	d_printf ("map_search searching for :'%s'", name);
	fstat (map_search_fd, &buf);

	blocksize = buf.st_size;
	blockstart = 0;

	/* search for name inside the data */
	do {
		realstart = maps_readpos (blocksize/2+blockstart, data, &j);
		if (realstart < 0) {
			d_printf ("map_search something went wrong. realstart < 0 name:%s", name);
			return 0;
		}
		d_printf ("search blocksize:%lld blockstart:%lld realstart:%lld j:%d", (long long int)blocksize, (long long int)blockstart, (long long int)realstart, j, data->name);
		if ((cmpres = strncasecmp (name, data->name, strlen (name)))> 0) {
			/* name is lager than data */
			blocksize = blocksize - (realstart+j-blockstart);
			blockstart = realstart+j+1;
		}
		else if (cmpres < 0) {
			/* name is smaller than data */
			blocksize = realstart - blockstart;
		}
		d_printf ("compare: '%s' with '%s'  returns:%d", name, data->name, cmpres);
	} while (cmpres != 0 && blocksize > MAP_SEARCHLINE_LEN*2);


	/* find first data inside list, go backwards until no match found */
	d_printf ("found blockstart:%lld blocksize:%lld, realstart:%lld  text:'%-12s'", (long long int)blockstart,
			(long long int)blocksize, (long long int)realstart, data->name);
	j = 0;
	if (cmpres <= 0) {
		blockstart = realstart;
		d_printf ("<=0 blockstart:%lld realstart:%lld name:%s", (long long int)blockstart,
				(long long int)realstart, name);
		do {
			realstart = maps_readpos (blockstart, data, &j);
			d_printf ("blockstart:%lld realstart:%lld data->name:%s", (long long int) blockstart, (long long int) realstart, data->name);
			blockstart = realstart-2;
			cmpres = strncasecmp (name, data->name, strlen (name));
			d_printf ("   cmpres:%d", cmpres);
		} while (blockstart >= 0 && cmpres <= 0 && realstart >= 0);
		blockstart = realstart+j;
	}
	else if (cmpres > 0) {
		blockstart = realstart;
		d_printf (">0 blockstart:%lld realstart:%lld name:%s", (long long int) blockstart, (long long int) realstart, name);
		do {
			realstart = maps_readpos (blockstart, data, &j);
			d_printf ("blockstart:%lld realstart:%lld data->name:%s", (long long int)blockstart, (long long int)realstart, data->name);
			blockstart = realstart+j;
			cmpres = strncasecmp (name, data->name, strlen (name));
			d_printf ("   cmpres:%d", cmpres);
		} while (cmpres > 0 && realstart >= 0);
		blockstart = realstart;
	}

	if (realstart < 0) return 0;

	if (blockstart < 0) blockstart = 0;
	for (i = 0, k = 0; i < datasize*2 && k < datasize; i++) {
		realstart = maps_readpos (blockstart, data+k, &j);
		d_printf ("create list: k:%d i:%d blockstart:%lld j:%d '%s'", k, i, (long long int)blockstart, j, data[k].name);
		if (realstart < 0) break;
		blockstart = realstart + j;
		if (strncasecmp (name, data[k].name, strlen (name)) == 0) k++;
	}

	return k;
};