How to create string token in C++ StringTokenizer.cpp


// Create a class called tokenizer that tokenizes a string.
#include <iostream>
#include <string>

using namespace std;

// The tokenizer class is used to tokenize a string.
// Pass the constructor the string to be tokenized.
// To obtain the next token, call get_token(),
// passing in a string that contains the delimiters.
class tokenizer {
	string s;
	string::size_type startidx;
	string::size_type endidx;

public:
	tokenizer(const string &str) {
		s = str;
		startidx = 0;
	}

	// Return a token from the string.
	string get_token(const string &delims);
};

// Return a token from the string. Return an
// empty string when no more tokens are found.
// Pass the delimiters in delims.
string tokenizer::get_token(const string &delims) {

	// Return an empty string when there are no more
	// tokens to return.
	if(startidx == string::npos) return string("");

	// Beginning at startidx, find the next delimiter.
	endidx = s.find_first_of(delims, startidx);

	// Construct a string that contains the token.
	string tok(s.substr(startidx, endidx-startidx));

	// Find the start of the next token. This is a
	// character that is not a delimiter.
	startidx = s.find_first_not_of(delims, endidx);

	// Return the next token.
	return tok;
}

int main()
{
	// Strings to be tokenized.
	string strA("I have four, five, six tokens. ");
	string strB("I might have more tokens!\nDo You?");

	// This string contains the delimiters.
	string delimiters(" ,.!?\n");

	// This string will hold the next token.
	string token;

	// Create two tokenizers.
	tokenizer tokA(strA);
	tokenizer tokB(strB);
	// Display the tokens in strA.
	cout << "The tokens in strA:\n";
	token = tokA.get_token(delimiters);
	while(token != "") {
		cout << token << endl;
		token = tokA.get_token(delimiters);
	}
	cout << endl;

	// Display the tokens in strB.
	cout << "The tokens in strB:\n";
	token = tokB.get_token(delimiters);
	while(token != "") {
		cout << token << endl;
		token = tokB.get_token(delimiters);
	}

	return 0;
}

Reference : Herb Schildt’s C++ Programming Cookbook

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: