Main Page Class Hierarchy Alphabetical List Compound List File List Compound Members File Members Related Pages

# XFuTokenizer.cpp

Go to the documentation of this file.

    00001 /*! \file
    00002  * X-Forge Util <br>
    00003  * Copyright 2000-2003 Fathammer Ltd
    00004  *
    00005  * \brief   String tokenizer utility
    00006  *
    00007  * $Id: XFuTokenizer.cpp,v 1.4 2003/08/12 13:34:08 lars Exp $
    00008  * $Date: 2003/08/12 13:34:08 $
    00009  * $Revision: 1.4 $
    00010  */
    00011
    00012 #include <xfcore/XFcCore.h>
    00013 #include <string.h> // memcpy
    00014 #include <xfutil/XFuTokenizer.h>
    00015
    00016
    00017 //! Constructor
    00018 XFuTokenizer::XFuTokenizer()
    00019 {
    00020     mData = NULL;
    00021     mCount=0;
    00022 }
    00023
    00024
    00025 //! Destructor
    00026 XFuTokenizer::~XFuTokenizer()
    00027 {
    00028     INT32 i;
    00029     for (i = 0; i < mCount; i++)
    00030         delete[] mData[i];
    00031     delete[] mData;
    00032 }
    00033
    00034
    00035 //! Tokenizes the input string, using the characters in 'separators' as delimiters for tokens
    00036 void XFuTokenizer::tokenize(const CHAR * aBuffer, const CHAR * aSeparators)
    00037 {
    00038     // if reusing this tokenizer, clean up old mess:
    00039     if (mData != NULL)
    00040     {
    00041         INT32 i;
    00042         for (i = 0; i < mCount; i++)
    00043         {
    00044             delete[] mData[i];
    00045         }
    00046         delete[] mData;
    00047         mCount = 0;
    00048         mData = NULL;
    00049     }
    00050     if (aBuffer == NULL) return;
    00051     INT32 sepcount = 0;
    00052     while (aSeparators[sepcount] != 0) sepcount++;
    00053
    00054     // pass 1: mCount tokens
    00055
    00056     mCount = 1;
    00057     INT32 i = 0;
    00058     while (aBuffer[i] != 0)
    00059     {
    00060         INT32 j;
    00061         for (j = 0; j < sepcount; j++)
    00062         {
    00063             if (aBuffer[i] == aSeparators[j])
    00064                 mCount++;
    00065         }
    00066         i++;
    00067     }
    00068     INT32 *offsets = new INT32[mCount + 1];
    00069     if (offsets == NULL)
    00070     {
    00071         mCount=0;
    00072         return;
    00073     }
    00074     INT32 *lengths = new INT32[mCount + 1];
    00075     if (lengths == NULL)
    00076     {
    00077         delete[] offsets;
    00078         mCount=0;
    00079         return;
    00080     }
    00081     mCount=0;
    00082     offsets[0]=0;
    00083     i=0;
    00084
    00085     // pass 2: mCount token sizes and offsets
    00086
    00087     while (aBuffer[i] != 0)
    00088     {
    00089         INT32 j;
    00090         for (j=0;j<sepcount;j++)
    00091         {
    00092             if (aBuffer[i]==aSeparators[j])
    00093                 offsets[++mCount]=i+1;
    00094         }
    00095         i++;
    00096     }
    00097     offsets[++mCount] = i + 1;
    00098
    00099     // pass 3: trim
    00100
    00101     for (i = 0; i < mCount; i++)
    00102     {
    00103         // ltrim
    00104         while (aBuffer[offsets[i]] == 32 || aBuffer[offsets[i]] == 9) offsets[i]++;
    00105         // rtrim
    00106         lengths[i] = offsets[i + 1] - (offsets[i] + 1);
    00107         while (lengths[i] > 0 && (aBuffer[offsets[i] + lengths[i] - 1] == 32 || aBuffer[offsets[i] + lengths[i] - 1] == 9)) lengths[i]--;
    00108     }
    00109
    00110     // pass 4: allocate
    00111
    00112     mData = new CHAR*[mCount];
    00113     if (mData == NULL)
    00114     {
    00115         delete[] offsets;
    00116         delete[] lengths;
    00117         mCount=0;
    00118         return;
    00119     }
    00120     for (i = 0; i < mCount; i++)
    00121     {
    00122         mData[i] = new CHAR[lengths[i] + 1];
    00123         mData[i][lengths[i]] = 0;
    00124     }
    00125
    00126     // pass 5: copy tokens
    00127
    00128     for (i = 0; i < mCount; i++)
    00129     {
    00130         INT32 j, c;
    00131         for (j = offsets[i], c = 0; j < offsets[i] + lengths[i]; j++, c++)
    00132         {
    00133             mData[i][c] = aBuffer[j];
    00134         }
    00135     }
    00136     delete[] offsets;
    00137     delete[] lengths;
    00138 }
    00139
    00140
    00141 //! Tokenizes the input string, using the ',' and '=' characters as delimiters for tokens
    00142 void XFuTokenizer::tokenize(const CHAR * aBuffer)
    00143 {
    00144     tokenize(aBuffer,"=,");
    00145 }
    00146
    00147
    00148 //! Case sensitive matching of a token and given string
    00149 INT XFuTokenizer::tokenEquals(INT32 aIdx, const CHAR * aCompareString)
    00150 {
    00151     const CHAR *t1 = getToken(aIdx), *t2 = aCompareString;
    00152     while (*t1 != 0 && *t2 != 0 && *t1 == *t2)
    00153     {
    00154         t1++;
    00155         t2++;
    00156     }
    00157     if (*t1 == *t2) return 1;
    00158     return 0;
    00159 }
    00160
    00161
    00162 //! Case insensitive matching of a token and given string
    00163 INT XFuTokenizer::tokenEqualsNocase(INT32 aIdx, const CHAR * aCompareString)
    00164 {
    00165     const CHAR *t1 = getToken(aIdx), *t2 = aCompareString;
    00166     while (*t1 != 0 && *t2 != 0 && upcase(*t1) == upcase(*t2))
    00167     {
    00168         t1++;
    00169         t2++;
    00170     }
    00171     if (*t1 == *t2) return 1;
    00172     return 0;
    00173 }
    00174
    00175
    00176 //! Get the token count
    00177 INT32 XFuTokenizer::getTokenCount()
    00178 {
    00179     return mCount;
    00180 }
    00181
    00182
    00183 //! Get a pointer to a token (as zero-terminated string)
    00184 const CHAR * XFuTokenizer::getToken(INT32 aIdx)
    00185 {
    00186     return (const CHAR*)mData[aIdx];
    00187 }
    00188
    00189
    00190 //! Get a duplicate of a token (as zero-terminated string)
    00191 CHAR * XFuTokenizer::duplicateToken(INT32 aIdx)
    00192 {
    00193     INT32 l=0;
    00194     const CHAR * s=mData[aIdx];
    00195     while (s[l]!=0) l++;
    00196     CHAR * copy=new CHAR[l+1];
    00197     copy[l]='\0';
    00198     memcpy(copy,s,l*sizeof(CHAR));
    00199     return copy;
    00200 }
    00201
    00202
    00203 //! upcase function
    00204 CHAR XFuTokenizer::upcase(CHAR i)
    00205 {
    00206     if (i>='a' && i<='z') return (CHAR)(i-('a'-'A'));
    00207     return i;
    00208 }
    00209
    00210
    00211

|  | X-Forge Documentation Confidential Copyright © 2002-2003 Fathammer |  | Documentation generated with doxygen by Dimitri van Heesch