X-Git-Url: http://git.maemo.org/git/?p=routino;a=blobdiff_plain;f=src%2Fwaysx.c;h=619fad6a8553158f30faa1b24120d69c9f569396;hp=3c4635b5c6b04dc15ed692a6b87023027c1f6d8d;hb=refs%2Fheads%2Fupstream;hpb=20283c6cf5c6951cc1f2787492c67a7fb72aee9a

diff --git a/src/waysx.c b/src/waysx.c
index 3c4635b..619fad6 100644
--- a/src/waysx.c
+++ b/src/waysx.c
@@ -1,5 +1,5 @@
 /***************************************
- $Header: /home/amb/routino/src/RCS/waysx.c,v 1.38 2010/05/22 18:40:47 amb Exp $
+ $Header: /home/amb/routino/src/RCS/waysx.c,v 1.52 2010/11/13 14:22:28 amb Exp $
 
  Extended Way data type functions.
 
@@ -24,18 +24,20 @@
 
 #include <assert.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
 
-#include "functions.h"
-#include "waysx.h"
 #include "ways.h"
 
+#include "waysx.h"
+
+#include "files.h"
+#include "logging.h"
+#include "functions.h"
 
-/* Variables */
 
-/*+ The command line '--slim' option. +*/
-extern int option_slim;
+/* Variables */
 
 /*+ The command line '--tmpdir' option or its default value. +*/
 extern char *option_tmpdirname;
@@ -45,11 +47,11 @@ static WaysX *sortwaysx;
 
 /* Functions */
 
+static int sort_by_id(WayX *a,WayX *b);
+static int sort_by_name_and_id(WayX *a,WayX *b);
 static int sort_by_name_and_prop_and_id(WayX *a,WayX *b);
-static int deduplicate_by_id(WayX *wayx,index_t index);
 
-static int sort_by_id(WayX *a,WayX *b);
-static int index_by_id(WayX *wayx,index_t index);
+static int deduplicate_and_index_by_id(WayX *wayx,index_t index);
 
 
 /*++++++++++++++++++++++++++++++++++++++
@@ -71,15 +73,15 @@ WaysX *NewWayList(int append)
  waysx->filename=(char*)malloc(strlen(option_tmpdirname)+32);
 
  if(append)
-    sprintf(waysx->filename,"%s/ways.input.tmp",option_tmpdirname);
+    sprintf(waysx->filename,"%s/waysx.input.tmp",option_tmpdirname);
  else
-    sprintf(waysx->filename,"%s/ways.%p.tmp",option_tmpdirname,waysx);
+    sprintf(waysx->filename,"%s/waysx.%p.tmp",option_tmpdirname,waysx);
 
  if(append)
    {
     off_t size,position=0;
 
-    waysx->fd=AppendFile(waysx->filename);
+    waysx->fd=OpenFileAppend(waysx->filename);
 
     size=SizeFile(waysx->filename);
 
@@ -97,7 +99,7 @@ WaysX *NewWayList(int append)
     SeekFile(waysx->fd,size);
    }
  else
-    waysx->fd=OpenFile(waysx->filename);
+    waysx->fd=OpenFileNew(waysx->filename);
 
  waysx->nfilename=(char*)malloc(strlen(option_tmpdirname)+32);
  sprintf(waysx->nfilename,"%s/waynames.%p.tmp",option_tmpdirname,waysx);
@@ -133,9 +135,7 @@ void FreeWayList(WaysX *waysx,int keep)
 
 
 /*++++++++++++++++++++++++++++++++++++++
-  Append a way to a way list.
-
-  void AppendWay Returns the newly appended way.
+  Append a single way to an unsorted way list.
 
   WaysX* waysx The set of ways to process.
 
@@ -151,8 +151,6 @@ void AppendWay(WaysX* waysx,way_t id,Way *way,const char *name)
  WayX wayx;
  FILESORT_VARINT size;
 
- assert(!waysx->idata);       /* Must not have idata filled in => unsorted */
-
  wayx.id=id;
  wayx.prop=0;
  wayx.way=*way;
@@ -164,6 +162,8 @@ void AppendWay(WaysX* waysx,way_t id,Way *way,const char *name)
  WriteFile(waysx->fd,name,strlen(name)+1);
 
  waysx->xnumber++;
+
+ assert(!(waysx->xnumber==0)); /* Zero marks the high-water mark for ways. */
 }
 
 
@@ -179,18 +179,12 @@ void SortWayList(WaysX* waysx)
  int fd,nfd;
  char *names[2]={NULL,NULL};
  int namelen[2]={0,0};
- int nnames=0,nprops=0;
+ int nnames=0;
  uint32_t lastlength=0;
- Way lastway;
-
- /* Check the start conditions */
-
- assert(!waysx->idata);         /* Must not have idata filled in => unsorted */
 
  /* Print the start message */
 
- printf("Sorting Ways");
- fflush(stdout);
+ printf_first("Sorting Ways by Name");
 
  /* Close the file and re-open it (finished appending) */
 
@@ -199,13 +193,11 @@ void SortWayList(WaysX* waysx)
 
  DeleteFile(waysx->filename);
 
- fd=OpenFile(waysx->filename);
-
- /* Sort the ways to allow compacting them and remove duplicates */
+ fd=OpenFileNew(waysx->filename);
 
- sortwaysx=waysx;
+ /* Sort the ways to allow separating the names */
 
- filesort_vary(waysx->fd,fd,(int (*)(const void*,const void*))sort_by_name_and_prop_and_id,(int (*)(void*,index_t))deduplicate_by_id);
+ filesort_vary(waysx->fd,fd,(int (*)(const void*,const void*))sort_by_name_and_id,NULL);
 
  /* Close the files */
 
@@ -214,14 +206,12 @@ void SortWayList(WaysX* waysx)
 
  /* Print the final message */
 
- printf("\rSorted Ways: Ways=%d Duplicates=%d\n",waysx->xnumber,waysx->xnumber-waysx->number);
- fflush(stdout);
+ printf_last("Sorted Ways by Name: Ways=%d",waysx->xnumber);
 
 
  /* Print the start message */
 
- printf("Compacting Ways: Ways=0 Names=0 Properties=0");
- fflush(stdout);
+ printf_first("Separating Way Names: Ways=0 Names=0");
 
  /* Open the files */
 
@@ -229,12 +219,12 @@ void SortWayList(WaysX* waysx)
 
  DeleteFile(waysx->filename);
 
- fd=OpenFile(waysx->filename);
- nfd=OpenFile(waysx->nfilename);
+ fd=OpenFileNew(waysx->filename);
+ nfd=OpenFileNew(waysx->nfilename);
 
- /* Copy from the single file into two files and index as we go. */
+ /* Copy from the single file into two files */
 
- for(i=0;i<waysx->number;i++)
+ for(i=0;i<waysx->xnumber;i++)
    {
     WayX wayx;
     FILESORT_VARINT size;
@@ -259,24 +249,10 @@ void SortWayList(WaysX* waysx)
 
     wayx.way.name=lastlength;
 
-    if(nprops==0 || wayx.way.name!=lastway.name || WaysCompare(&lastway,&wayx.way))
-      {
-       lastway=wayx.way;
-
-       waysx->cnumber++;
-
-       nprops++;
-      }
-
-    wayx.prop=nprops-1;
-
     WriteFile(fd,&wayx,sizeof(WayX));
 
     if(!((i+1)%10000))
-      {
-       printf("\rCompacting Ways: Ways=%d Names=%d Properties=%d",i+1,nnames,nprops);
-       fflush(stdout);
-      }
+       printf_middle("Separating Way Names: Ways=%d Names=%d",i+1,nnames);
    }
 
  if(names[0]) free(names[0]);
@@ -293,14 +269,12 @@ void SortWayList(WaysX* waysx)
 
  /* Print the final message */
 
- printf("\rCompacted Ways: Ways=%d Names=%d Properties=%d \n",waysx->number,nnames,nprops);
- fflush(stdout);
+ printf_last("Separated Way Names: Ways=%d Names=%d ",waysx->xnumber,nnames);
 
 
  /* Print the start message */
 
- printf("Sorting Ways");
- fflush(stdout);
+ printf_first("Sorting Ways");
 
  /* Open the files */
 
@@ -308,19 +282,136 @@ void SortWayList(WaysX* waysx)
 
  DeleteFile(waysx->filename);
 
- fd=OpenFile(waysx->filename);
+ fd=OpenFileNew(waysx->filename);
 
  /* Allocate the array of indexes */
 
- waysx->idata=(way_t*)malloc(waysx->number*sizeof(way_t));
+ waysx->idata=(way_t*)malloc(waysx->xnumber*sizeof(way_t));
 
  assert(waysx->idata); /* Check malloc() worked */
 
  /* Sort the ways by index and index them */
 
+ waysx->number=0;
+
  sortwaysx=waysx;
 
- filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,(int (*)(void*,index_t))index_by_id);
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,(int (*)(void*,index_t))deduplicate_and_index_by_id);
+
+ /* Close the files and re-open them */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ /* Print the final message */
+
+ printf_last("Sorted Ways: Ways=%d Duplicates=%d",waysx->number,waysx->xnumber-waysx->number);
+}
+
+
+/*++++++++++++++++++++++++++++++++++++++
+  Compact the list of ways.
+
+  WaysX* waysx The set of ways to process.
+  ++++++++++++++++++++++++++++++++++++++*/
+
+void CompactWayList(WaysX* waysx)
+{
+ index_t i;
+ int fd;
+ Way lastway;
+
+ /* Print the start message */
+
+ printf_first("Sorting Ways by Properties");
+
+ /* Close the file and re-open it */
+
+ CloseFile(waysx->fd);
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Sort the ways to allow compacting according to he properties */
+
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_name_and_prop_and_id,NULL);
+
+ /* Close the files */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ /* Print the final message */
+
+ printf_last("Sorted Ways by Properties: Ways=%d",waysx->number);
+
+
+ /* Print the start message */
+
+ printf_first("Compacting Ways: Ways=0 Properties=0");
+
+ /* Open the files */
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Update the way as we go using the sorted index */
+
+ waysx->cnumber=0;
+
+ for(i=0;i<waysx->number;i++)
+   {
+    WayX wayx;
+
+    ReadFile(waysx->fd,&wayx,sizeof(WayX));
+
+    if(waysx->cnumber==0 || wayx.way.name!=lastway.name || WaysCompare(&lastway,&wayx.way))
+      {
+       lastway=wayx.way;
+
+       waysx->cnumber++;
+      }
+
+    wayx.prop=waysx->cnumber-1;
+
+    WriteFile(fd,&wayx,sizeof(WayX));
+
+    if(!((i+1)%10000))
+       printf_middle("Compacting Ways: Ways=%d Properties=%d",i+1,waysx->cnumber);
+   }
+
+ /* Close the files */
+
+ CloseFile(waysx->fd);
+ CloseFile(fd);
+
+ /* Print the final message */
+
+ printf_last("Compacted Ways: Ways=%d Properties=%d ",waysx->number,waysx->cnumber);
+
+
+ /* Print the start message */
+
+ printf_first("Sorting Ways");
+
+ /* Open the files */
+
+ waysx->fd=ReOpenFile(waysx->filename);
+
+ DeleteFile(waysx->filename);
+
+ fd=OpenFileNew(waysx->filename);
+
+ /* Sort the ways by index */
+
+ filesort_fixed(waysx->fd,fd,sizeof(WayX),(int (*)(const void*,const void*))sort_by_id,NULL);
 
  /* Close the files and re-open them */
 
@@ -331,8 +422,7 @@ void SortWayList(WaysX* waysx)
 
  /* Print the final message */
 
- printf("\rSorted Ways: Ways=%d\n",waysx->number);
- fflush(stdout);
+ printf_last("Sorted Ways: Ways=%d",waysx->number);
 }
 
 
@@ -361,16 +451,16 @@ static int sort_by_id(WayX *a,WayX *b)
 
 
 /*++++++++++++++++++++++++++++++++++++++
-  Sort the ways into name, properties and id order.
+  Sort the ways into name and id order.
 
-  int sort_by_name_and_prop_and_id Returns the comparison of the name, properties and id fields.
+  int sort_by_name_and_id Returns the comparison of the name and id fields.
 
   WayX *a The first extended Way.
 
   WayX *b The second extended Way.
   ++++++++++++++++++++++++++++++++++++++*/
 
-static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
+static int sort_by_name_and_id(WayX *a,WayX *b)
 {
  int compare;
  char *a_name=(char*)a+sizeof(WayX);
@@ -381,6 +471,31 @@ static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
  if(compare)
     return(compare);
 
+ return(sort_by_id(a,b));
+}
+
+
+/*++++++++++++++++++++++++++++++++++++++
+  Sort the ways into name, properties and id order.
+
+  int sort_by_name_and_prop_and_id Returns the comparison of the name, properties and id fields.
+
+  WayX *a The first extended Way.
+
+  WayX *b The second extended Way.
+  ++++++++++++++++++++++++++++++++++++++*/
+
+static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
+{
+ int compare;
+ index_t a_name=a->way.name;
+ index_t b_name=b->way.name;
+
+ if(a_name<b_name)
+    return(-1);
+ else if(a_name>b_name)
+    return(1);
+
  compare=WaysCompare(&a->way,&b->way);
 
  if(compare)
@@ -391,16 +506,16 @@ static int sort_by_name_and_prop_and_id(WayX *a,WayX *b)
 
 
 /*++++++++++++++++++++++++++++++++++++++
-  Deduplicate the extended ways using the id after sorting.
+  Deduplicate the extended ways using the id after sorting and create the index.
 
-  int deduplicate_by_id Return 1 if the value is to be kept, otherwise zero.
+  int deduplicate_and_index_by_id Return 1 if the value is to be kept, otherwise zero.
 
   WayX *wayx The extended way.
 
   index_t index The index of this way in the total.
   ++++++++++++++++++++++++++++++++++++++*/
 
-static int deduplicate_by_id(WayX *wayx,index_t index)
+static int deduplicate_and_index_by_id(WayX *wayx,index_t index)
 {
  static way_t previd;
 
@@ -410,6 +525,8 @@ static int deduplicate_by_id(WayX *wayx,index_t index)
 
     sortwaysx->number++;
 
+    sortwaysx->idata[index]=wayx->id;
+
     return(1);
    }
 
@@ -418,24 +535,6 @@ static int deduplicate_by_id(WayX *wayx,index_t index)
 
 
 /*++++++++++++++++++++++++++++++++++++++
-  Index the ways after sorting.
-
-  int index_by_id Return 1 if the value is to be kept, otherwise zero.
-
-  WayX *wayx The extended way.
-
-  index_t index The index of this way in the total.
-  ++++++++++++++++++++++++++++++++++++++*/
-
-static int index_by_id(WayX *wayx,index_t index)
-{
- sortwaysx->idata[index]=wayx->id;
-
- return(1);
-}
-
-
-/*++++++++++++++++++++++++++++++++++++++
   Find a particular way index.
 
   index_t IndexWayX Returns the index of the extended way with the specified id.
@@ -451,8 +550,6 @@ index_t IndexWayX(WaysX* waysx,way_t id)
  int end=waysx->number-1;
  int mid;
 
- assert(waysx->idata);         /* Must have idata filled in => sorted */
-
  /* Binary search - search key exact match only is required.
   *
   *  # <- start  |  Check mid and move start or end if it doesn't match
@@ -497,37 +594,6 @@ index_t IndexWayX(WaysX* waysx,way_t id)
 
 
 /*++++++++++++++++++++++++++++++++++++++
-  Lookup a particular way.
-
-  WayX *LookupWayX Returns a pointer to the extended way with the specified id.
-
-  WaysX* waysx The set of ways to process.
-
-  index_t index The way index to look for.
-
-  int position The position in the cache to use.
-  ++++++++++++++++++++++++++++++++++++++*/
-
-WayX *LookupWayX(WaysX* waysx,index_t index,int position)
-{
- assert(index!=NO_WAY);     /* Must be a valid way */
-
- if(option_slim)
-   {
-    SeekFile(waysx->fd,index*sizeof(WayX));
-
-    ReadFile(waysx->fd,&waysx->cached[position-1],sizeof(WayX));
-
-    return(&waysx->cached[position-1]);
-   }
- else
-   {
-    return(&waysx->xdata[index]);
-   }
-}
-
-
-/*++++++++++++++++++++++++++++++++++++++
   Save the way list to a file.
 
   WaysX* waysx The set of ways to save.
@@ -540,59 +606,49 @@ void SaveWayList(WaysX* waysx,const char *filename)
  index_t i;
  int fd,nfd;
  int position=0;
- Ways *ways;
-
- printf("Writing Ways: Ways=0");
- fflush(stdout);
-
- if(!option_slim)
-    waysx->xdata=MapFile(waysx->filename);
-
- /* Fill in a Ways structure with the offset of the real data in the file after
-    the Way structure itself. */
+ WaysFile waysfile={0};
+ allow_t allow=0;
+ wayprop_t  props=0;
 
- ways=calloc(1,sizeof(Ways));
+ /* Print the start message */
 
- assert(ways); /* Check calloc() worked */
+ printf_first("Writing Ways: Ways=0");
 
- ways->number=waysx->cnumber;
- ways->onumber=waysx->number;
+ /* Map into memory */
 
- ways->allow=0;
- ways->props=0;
+#if !SLIM
+ waysx->xdata=MapFile(waysx->filename);
+#endif
 
- ways->data=NULL;
- ways->ways=NULL;
- ways->names=NULL;
+ /* Write out the ways data */
 
- /* Write out the Ways structure and then the real data. */
+ fd=OpenFileNew(filename);
 
- fd=OpenFile(filename);
+ SeekFile(fd,sizeof(WaysFile));
 
  for(i=0;i<waysx->number;i++)
    {
     WayX *wayx=LookupWayX(waysx,i,1);
 
-    ways->allow|=wayx->way.allow;
-    ways->props|=wayx->way.props;
+    allow|=wayx->way.allow;
+    props|=wayx->way.props;
 
-    SeekFile(fd,sizeof(Ways)+wayx->prop*sizeof(Way));
+    SeekFile(fd,sizeof(WaysFile)+(off_t)wayx->prop*sizeof(Way));
     WriteFile(fd,&wayx->way,sizeof(Way));
 
     if(!((i+1)%10000))
-      {
-       printf("\rWriting Ways: Ways=%d",i+1);
-       fflush(stdout);
-      }
+       printf_middle("Writing Ways: Ways=%d",i+1);
    }
 
- SeekFile(fd,0);
- WriteFile(fd,ways,sizeof(Ways));
+ /* Unmap from memory */
+
+#if !SLIM
+ waysx->xdata=UnmapFile(waysx->filename);
+#endif
 
- if(!option_slim)
-    waysx->xdata=UnmapFile(waysx->filename);
+ /* Write out the ways names */
 
- SeekFile(fd,sizeof(Ways)+ways->number*sizeof(Way));
+ SeekFile(fd,sizeof(WaysFile)+(off_t)waysx->cnumber*sizeof(Way));
 
  nfd=ReOpenFile(waysx->nfilename);
 
@@ -612,12 +668,20 @@ void SaveWayList(WaysX* waysx,const char *filename)
 
  CloseFile(nfd);
 
- CloseFile(fd);
+ /* Write out the header structure */
 
- printf("\rWrote Ways: Ways=%d  \n",waysx->number);
- fflush(stdout);
+ waysfile.number=waysx->cnumber;
+ waysfile.onumber=waysx->number;
 
- /* Free the fake Ways */
+ waysfile.allow=allow;
+ waysfile.props=props;
+
+ SeekFile(fd,0);
+ WriteFile(fd,&waysfile,sizeof(WaysFile));
+
+ CloseFile(fd);
+
+ /* Print the final message */
 
- free(ways);
+ printf_last("Wrote Ways: Ways=%d",waysx->number);
 }