<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>
<!DOCTYPE MAN SYSTEM "../../manrev.dtd">
<MAN>
  <LANGUAGE>eng</LANGUAGE>
  <TITLE>tokens</TITLE>
  <TYPE>Scilab Function</TYPE>
  <DATE>April 1993</DATE>
  <SHORT_DESCRIPTION name="tokens"> returns the tokens of a character string.  </SHORT_DESCRIPTION>
  <CALLING_SEQUENCE>
    <CALLING_SEQUENCE_ITEM>T=tokens(str [,delimiter])  </CALLING_SEQUENCE_ITEM>
  </CALLING_SEQUENCE>
  <PARAM>
    <PARAM_INDENT>
      <PARAM_ITEM>
        <PARAM_NAME>str</PARAM_NAME>
        <PARAM_DESCRIPTION>
          <SP>: a character string. The string where to search the tokens.</SP>
        </PARAM_DESCRIPTION>
      </PARAM_ITEM>
      <PARAM_ITEM>
        <PARAM_NAME>delimiter</PARAM_NAME>
        <PARAM_DESCRIPTION>
          <SP>: (optional) a character or a vector of characters. The tokens delimeters.</SP>
        </PARAM_DESCRIPTION>
      </PARAM_ITEM>
      <PARAM_ITEM>
        <PARAM_NAME>T</PARAM_NAME>
        <PARAM_DESCRIPTION>
          <SP>: column vector of found tokens</SP>
        </PARAM_DESCRIPTION>
      </PARAM_ITEM>
    </PARAM_INDENT>
  </PARAM>
  <DESCRIPTION>
    <P><VERB>T=tokens(str [,delimiter]) </VERB> searches the tokens included in the
    string <VERB>str</VERB>. The <VERB>delimiter</VERB> default value is [&quot; &quot;,&lt;Tab&gt;]
    where &lt;Tab&gt; stands for <VERB>ascii(9)</VERB>.</P>
  </DESCRIPTION>
  <EXAMPLE>
<![CDATA[
tokens('This is a character string')

tokens('SCI/demos/scicos','/')

tokens('y=a+b*2',['=','+','*'])
 ]]>
  </EXAMPLE>
  <SEE_ALSO>
    <SEE_ALSO_ITEM>
      <LINK>strindex</LINK>
    </SEE_ALSO_ITEM>
    <SEE_ALSO_ITEM>
      <LINK>tokenpos</LINK>
    </SEE_ALSO_ITEM>
  </SEE_ALSO>
</MAN>
