/***************************************
- $Header: /home/amb/routino/src/RCS/waysx.c,v 1.38 2010/05/22 18:40:47 amb Exp $
+ $Header: /home/amb/routino/src/RCS/waysx.c,v 1.51 2010/09/19 16:17:45 amb Exp $
Extended Way data type functions.
#include <assert.h>
#include <stdlib.h>
+#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
-#include "functions.h"
-#include "waysx.h"
#include "ways.h"
+#include "waysx.h"
+
+#include "files.h"
+#include "functions.h"
-/* Variables */
-/*+ The command line '--slim' option. +*/
-extern int option_slim;
+/* Variables */
/*+ The command line '--tmpdir' option or its default value. +*/
extern char *option_tmpdirname;
/* Functions */
+static int sort_by_id(WayX *a,WayX *b);
+static int sort_by_name_and_id(WayX *a,WayX *b);
static int sort_by_name_and_prop_and_id(WayX *a,WayX *b);
-static int deduplicate_by_id(WayX *wayx,index_t index);
-static int sort_by_id(WayX *a,WayX *b);
-static int index_by_id(WayX *wayx,index_t index);
+static int deduplicate_and_index_by_id(WayX *wayx,index_t index);
/*++++++++++++++++++++++++++++++++++++++
waysx->filename=(char*)malloc(strlen(option_tmpdirname)+32);
if(append)
- sprintf(waysx->filename,"%s/ways.input.tmp",option_tmpdirname);
+ sprintf(waysx->filename,"%s/waysx.input.tmp",option_tmpdirname);
else
- sprintf(waysx->filename,"%s/ways.%p.tmp",option_tmpdirname,waysx);
+ sprintf(waysx->filename,"%s/waysx.%p.tmp",option_tmpdirname,waysx);
if(append)
{
off_t size,position=0;
- waysx->fd=AppendFile(waysx->filename);
+ waysx->fd=OpenFileAppend(waysx->filename);
size=SizeFile(waysx->filename);
SeekFile(waysx->fd,size);
}
else
- waysx->fd=OpenFile(waysx->filename);
+ waysx->fd=OpenFileNew(waysx->filename);
waysx->nfilename=(char*)malloc(strlen(option_tmpdirname)+32);
sprintf(waysx->nfilename,"%s/waynames.%p.tmp",option_tmpdirname,waysx);
/*++++++++++++++++++++++++++++++++++++++
- Append a way to a way list.
-
- void AppendWay Returns the newly appended way.
+ Append a single way to an unsorted way list.
WaysX* waysx The set of ways to process.
WayX wayx;
FILESORT_VARINT size;
- assert(!waysx->idata); /* Must not have idata filled in => unsorted */
-
wayx.id=id;
wayx.prop=0;
wayx.way=*way;
WriteFile(waysx->fd,name,strlen(name)+1);
waysx->xnumber++;
+
+ assert(!(waysx->xnumber==0)); /* Zero marks the high-water mark for ways. */
}
int fd,nfd;
char *names[2]={NULL,NULL};
int namelen[2]={0,0};
- int nnames=0,nprops=0;
+ int nnames=0;
uint32_t lastlength=0;
- Way lastway;
-
- /* Check the start conditions */
-
- assert(!waysx->idata); /* Must not have idata filled in => unsorted */
/* Print the start message */
- printf("Sorting Ways");
+ printf("Sorting Ways by Name");
fflush(stdout);
/* Close the file and re-open it (finished appending) */
DeleteFile(waysx->filename);
- fd=OpenFile(waysx->filename);
+ fd=OpenFileNew(waysx->filename);
- /* Sort the ways to allow compacting them and remove duplicates */
+ /* Sort the ways to allow separating the names */
- sortwaysx=waysx;
-
- filesort_vary(waysx->fd,fd,(int (*)(const void*,const void*))sort_by_name_and_prop_and_id,(int (*)(void*,index_t))deduplicate_by_id);
+ filesort_vary(waysx->fd,fd,(int (*)(const void*,const void*))sort_by_name_and_id,NULL);
/* Close the files */
/* Print the final message */
- printf("\rSorted Ways: Ways=%d Duplicates=%d\n",waysx->xnumber,waysx->xnumber-waysx->number);
+ printf("\rSorted Ways by Name: Ways=%d\n",waysx->xnumber);
fflush(stdout);
/* Print the start message */
- printf("Compacting Ways: Ways=0 Names=0 Properties=0");
+ printf("Separating Way Names: Ways=0 Names=0");
fflush(stdout);
/* Open the files */
DeleteFile(waysx->filename);
- fd=OpenFile(waysx->filename);
- nfd=OpenFile(waysx->nfilename);
+ fd=OpenFileNew(waysx->filename);
+ nfd=OpenFileNew(waysx->nfilename);
- /* Copy from the single file into two files and index as we go. */
+ /* Copy from the single file into two files */
- for(i=0;i<waysx->number;i++)
+ for(i=0;i<waysx->xnumber;i++)
{
WayX wayx;
FILESORT_VARINT size;
wayx.way.name=lastlength;
- if(nprops==0 || wayx.way.name!=lastway.name || WaysCompare(&lastway,&wayx.way))
- {
- lastway=wayx.way;
-
- waysx->cnumber++;
-
- nprops++;
- }
-
- wayx.prop=nprops-1;
-
WriteFile(fd,&wayx,sizeof(WayX));
if(!((i+1)%10000))
{
- printf("\rCompacting Ways: Ways=%d Names=%d Properties=%d",i+1,nnames,nprops);
+ printf("\rSeparating Way Names: Ways=%d Names=%d",i+1,nnames);
fflush(stdout);
}
}
/* Print the final message */
- printf("\rCompacted Ways: Ways=%d Names=%d Properties=%d \n",waysx->number,nnames,nprops);
+ printf("\rSeparated Way Names: Ways=%d Names=%d \n",waysx->xnumber,nnames);
fflush(stdout);
DeleteFile(waysx->filename);
- fd=OpenFile(waysx->filename);
+ fd=OpenFileNew(waysx->filename);
/* Allocate the array of indexes */
- waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
+ waysx->idata=(way_t*)malloc(waysx->xnumber*sizeof(way_t));
assert(waysx->idata); /* Check malloc() worked */
/* Sort the ways by index and index them */
+ waysx->number=0;
+
sortwaysx=waysx;
- filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,(int (*)(void*,index_t))index_by_id);
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,(int (*)(void*,index_t))deduplicate_and_index_by_id);
+
+ /* Close the files and re-open them */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ /* Print the final message */
+
+ printf("\rSorted Ways: Ways=%d Duplicates=%d\n",waysx->number,waysx->xnumber-waysx->number);
+ fflush(stdout);
+}
+
+
+/*++++++++++++++++++++++++++++++++++++++
+ Compact the list of ways.
+
+ WaysX* waysx The set of ways to process.
+ ++++++++++++++++++++++++++++++++++++++*/
+
+void CompactWayList(WaysX* waysx)
+{
+ index_t i;
+ int fd;
+ Way lastway;
+
+ /* Print the start message */
+
+ printf("Sorting Ways by Properties");
+ fflush(stdout);
+
+ /* Close the file and re-open it */
+
+ CloseFile(waysx->fd);
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Sort the ways to allow compacting according to he properties */
+
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_name_and_prop_and_id,NULL);
+
+ /* Close the files */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ /* Print the final message */
+
+ printf("\rSorted Ways by Properties: Ways=%d\n",waysx->number);
+ fflush(stdout);
+
+
+ /* Print the start message */
+
+ printf("Compacting Ways: Ways=0 Properties=0");
+ fflush(stdout);
+
+ /* Open the files */
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Update the way as we go using the sorted index */
+
+ waysx->cnumber=0;
+
+ for(i=0;i<waysx->number;i++)
+ {
+ WayX wayx;
+
+ ReadFile(waysx->fd,&wayx,sizeof(WayX));
+
+ if(waysx->cnumber==0 || wayx.way.name!=lastway.name || WaysCompare(&lastway,&wayx.way))
+ {
+ lastway=wayx.way;
+
+ waysx->cnumber++;
+ }
+
+ wayx.prop=waysx->cnumber-1;
+
+ WriteFile(fd,&wayx,sizeof(WayX));
+
+ if(!((i+1)%10000))
+ {
+ printf("\rCompacting Ways: Ways=%d Properties=%d",i+1,waysx->cnumber);
+ fflush(stdout);
+ }
+ }
+
+ /* Close the files */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ /* Print the final message */
+
+ printf("\rCompacted Ways: Ways=%d Properties=%d \n",waysx->number,waysx->cnumber);
+ fflush(stdout);
+
+
+ /* Print the start message */
+
+ printf("Sorting Ways");
+ fflush(stdout);
+
+ /* Open the files */
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Sort the ways by index */
+
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,NULL);
/* Close the files and re-open them */
/*++++++++++++++++++++++++++++++++++++++
- Sort the ways into name, properties and id order.
+ Sort the ways into name and id order.
- int sort_by_name_and_prop_and_id Returns the comparison of the name, properties and id fields.
+ int sort_by_name_and_id Returns the comparison of the name and id fields.
WayX *a The first extended Way.
WayX *b The second extended Way.
++++++++++++++++++++++++++++++++++++++*/
-static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
+static int sort_by_name_and_id(WayX *a,WayX *b)
{
int compare;
char *a_name=(char*)a+sizeof(WayX);
if(compare)
return(compare);
+ return(sort_by_id(a,b));
+}
+
+
+/*++++++++++++++++++++++++++++++++++++++
+ Sort the ways into name, properties and id order.
+
+ int sort_by_name_and_prop_and_id Returns the comparison of the name, properties and id fields.
+
+ WayX *a The first extended Way.
+
+ WayX *b The second extended Way.
+ ++++++++++++++++++++++++++++++++++++++*/
+
+static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
+{
+ int compare;
+ index_t a_name=a->way.name;
+ index_t b_name=b->way.name;
+
+ if(a_name<b_name)
+ return(-1);
+ else if(a_name>b_name)
+ return(1);
+
compare=WaysCompare(&a->way,&b->way);
if(compare)
/*++++++++++++++++++++++++++++++++++++++
- Deduplicate the extended ways using the id after sorting.
+ Deduplicate the extended ways using the id after sorting and create the index.
- int deduplicate_by_id Return 1 if the value is to be kept, otherwise zero.
+ int deduplicate_and_index_by_id Return 1 if the value is to be kept, otherwise zero.
WayX *wayx The extended way.
index_t index The index of this way in the total.
++++++++++++++++++++++++++++++++++++++*/
-static int deduplicate_by_id(WayX *wayx,index_t index)
+static int deduplicate_and_index_by_id(WayX *wayx,index_t index)
{
static way_t previd;
sortwaysx->number++;
+ sortwaysx->idata[index]=wayx->id;
+
return(1);
}
/*++++++++++++++++++++++++++++++++++++++
- Index the ways after sorting.
-
- int index_by_id Return 1 if the value is to be kept, otherwise zero.
-
- WayX *wayx The extended way.
-
- index_t index The index of this way in the total.
- ++++++++++++++++++++++++++++++++++++++*/
-
-static int index_by_id(WayX *wayx,index_t index)
-{
- sortwaysx->idata[index]=wayx->id;
-
- return(1);
-}
-
-
-/*++++++++++++++++++++++++++++++++++++++
Find a particular way index.
index_t IndexWayX Returns the index of the extended way with the specified id.
int end=waysx->number-1;
int mid;
- assert(waysx->idata); /* Must have idata filled in => sorted */
-
/* Binary search - search key exact match only is required.
*
* # <- start | Check mid and move start or end if it doesn't match
/*++++++++++++++++++++++++++++++++++++++
- Lookup a particular way.
-
- WayX *LookupWayX Returns a pointer to the extended way with the specified id.
-
- WaysX* waysx The set of ways to process.
-
- index_t index The way index to look for.
-
- int position The position in the cache to use.
- ++++++++++++++++++++++++++++++++++++++*/
-
-WayX *LookupWayX(WaysX* waysx,index_t index,int position)
-{
- assert(index!=NO_WAY); /* Must be a valid way */
-
- if(option_slim)
- {
- SeekFile(waysx->fd,index*sizeof(WayX));
-
- ReadFile(waysx->fd,&waysx->cached[position-1],sizeof(WayX));
-
- return(&waysx->cached[position-1]);
- }
- else
- {
- return(&waysx->xdata[index]);
- }
-}
-
-
-/*++++++++++++++++++++++++++++++++++++++
Save the way list to a file.
WaysX* waysx The set of ways to save.
index_t i;
int fd,nfd;
int position=0;
- Ways *ways;
+ WaysFile waysfile={0};
+ allow_t allow=0;
+ wayprop_t props=0;
+
+ /* Print the start message */
printf("Writing Ways: Ways=0");
fflush(stdout);
- if(!option_slim)
- waysx->xdata=MapFile(waysx->filename);
-
- /* Fill in a Ways structure with the offset of the real data in the file after
- the Way structure itself. */
-
- ways=calloc(1,sizeof(Ways));
-
- assert(ways); /* Check calloc() worked */
+ /* Map into memory */
- ways->number=waysx->cnumber;
- ways->onumber=waysx->number;
+#if !SLIM
+ waysx->xdata=MapFile(waysx->filename);
+#endif
- ways->allow=0;
- ways->props=0;
+ /* Write out the ways data */
- ways->data=NULL;
- ways->ways=NULL;
- ways->names=NULL;
+ fd=OpenFileNew(filename);
- /* Write out the Ways structure and then the real data. */
-
- fd=OpenFile(filename);
+ SeekFile(fd,sizeof(WaysFile));
for(i=0;i<waysx->number;i++)
{
WayX *wayx=LookupWayX(waysx,i,1);
- ways->allow|=wayx->way.allow;
- ways->props|=wayx->way.props;
+ allow|=wayx->way.allow;
+ props|=wayx->way.props;
- SeekFile(fd,sizeof(Ways)+wayx->prop*sizeof(Way));
+ SeekFile(fd,sizeof(WaysFile)+(off_t)wayx->prop*sizeof(Way));
WriteFile(fd,&wayx->way,sizeof(Way));
if(!((i+1)%10000))
}
}
- SeekFile(fd,0);
- WriteFile(fd,ways,sizeof(Ways));
+ /* Unmap from memory */
+
+#if !SLIM
+ waysx->xdata=UnmapFile(waysx->filename);
+#endif
- if(!option_slim)
- waysx->xdata=UnmapFile(waysx->filename);
+ /* Write out the ways names */
- SeekFile(fd,sizeof(Ways)+ways->number*sizeof(Way));
+ SeekFile(fd,sizeof(WaysFile)+(off_t)waysx->cnumber*sizeof(Way));
nfd=ReOpenFile(waysx->nfilename);
CloseFile(nfd);
+ /* Write out the header structure */
+
+ waysfile.number=waysx->cnumber;
+ waysfile.onumber=waysx->number;
+
+ waysfile.allow=allow;
+ waysfile.props=props;
+
+ SeekFile(fd,0);
+ WriteFile(fd,&waysfile,sizeof(WaysFile));
+
CloseFile(fd);
+ /* Print the final message */
+
printf("\rWrote Ways: Ways=%d \n",waysx->number);
fflush(stdout);
-
- /* Free the fake Ways */
-
- free(ways);
}
projects / routino / blobdiff